diff --git a/.gitignore b/.gitignore index da611dcf..369f71c6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ *~ -*.swp -*.swo +*.sw? *.pyc .ropeproject .tox diff --git a/.travis.yml b/.travis.yml index 876af276..b552eabf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,12 @@ language: python +python: 3.5 sudo: false env: - TOXENV=py26 - TOXENV=py27 - - TOXENV=py32 - TOXENV=py33 - TOXENV=py34 + - TOXENV=py35 - TOXENV=pypy - TOXENV=cov - TOXENV=sith @@ -14,8 +15,9 @@ matrix: - env: TOXENV=cov - env: TOXENV=sith - env: TOXENV=pypy +python: 3.5 install: - - pip install --quiet --use-mirrors tox + - pip install --quiet tox 'virtualenv<14.0.0' 'pip<8.0.0' script: - tox after_script: @@ -23,3 +25,4 @@ after_script: pip install --quiet --use-mirrors coveralls; coveralls; fi + diff --git a/AUTHORS.txt b/AUTHORS.txt index bc258d6b..9e7f9800 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -34,5 +34,9 @@ Phillip Berndt (@phillipberndt) Ian Lee (@IanLee1521) Farkhad Khatamov (@hatamov) Kevin Kelley (@kelleyk) +Sid Shanker (@squidarth) +Reinoud Elhorst (@reinhrst) +Guido van Rossum (@gvanrossum) +Dmytro Sadovnychyi (@sadovnychyi) Note: (@user) means a github user name. diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 13c218d6..6adf8d72 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,14 @@ Changelog --------- +0.10.0 (2016-06-) ++++++++++++++++++ + +- Actual semantic completions for the complete Python syntax. +- Basic type inference for ``yield from`` PEP 380. +- PEP 484 support (most of the important features of it). Thanks Claude! (@reinhrst) +- Again a lot of internal changes. + 0.9.0 (2015-04-10) ++++++++++++++++++ diff --git a/LICENSE.txt b/LICENSE.txt index 12f223fc..9492c4e8 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,3 +1,15 @@ +All contributions towards Jedi are MIT licensed. + +Some Python files have been taken from the standard library and are therefore +PSF licensed. Modifications on these files are dual licensed (both MIT and +PSF). These files are: + +- jedi/parser/pgen2 +- jedi/parser/tokenize.py +- jedi/parser/token.py +- test/test_parser/test_pgen2.py + +------------------------------------------------------------------------------- The MIT License (MIT) Copyright (c) <2013> @@ -19,3 +31,52 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------- + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014, 2015 Python Software Foundation; All Rights Reserved" +are retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/README.rst b/README.rst index 17f1aba8..1adf138f 100644 --- a/README.rst +++ b/README.rst @@ -30,19 +30,23 @@ implementation as a `VIM-Plugin `_, which uses Jedi's autocompletion. We encourage you to use Jedi in your IDEs. It's really easy. -Jedi can currently be used with the following editors: +Jedi can currently be used with the following editors/projects: -- Vim (jedi-vim_, YouCompleteMe_) -- Emacs (Jedi.el_, elpy_, anaconda-mode_, ycmd_) +- Vim (jedi-vim_, YouCompleteMe_, deoplete-jedi_) +- Emacs (Jedi.el_, company-mode_, elpy_, anaconda-mode_, ycmd_) - Sublime Text (SublimeJEDI_ [ST2 + ST3], anaconda_ [only ST3]) -- SynWrite_ - TextMate_ (Not sure if it's actually working) - Kate_ version 4.13+ supports it natively, you have to enable it, though. [`proof `_] - -And it powers the following projects: - +- Atom_ (autocomplete-python_) +- SourceLair_ +- `GNOME Builder`_ (with support for GObject Introspection) +- `Visual Studio Code`_ (via `Python Extension `_) +- Gedit (gedi_) - wdb_ - Web Debugger +- `Eric IDE`_ (Available as a plugin) + +and many more! Here are some pictures taken from jedi-vim_: @@ -91,11 +95,11 @@ understands, see: `Features `_. A list of caveats can be found on the same page. -You can run Jedi on cPython 2.6, 2.7, 3.2, 3.3 or 3.4, but it should also +You can run Jedi on cPython 2.6, 2.7, 3.3, 3.4 or 3.5 but it should also understand/parse code older than those versions. Tips on how to use Jedi efficiently can be found `here -`_. +`_. API --- @@ -173,15 +177,34 @@ For more detailed information visit the `testing documentation `_ +Acknowledgements +================ + +- Takafumi Arakaki (@tkf) for creating a solid test environment and a lot of + other things. +- Danilo Bargen (@dbrgn) for general housekeeping and being a good friend :). +- Guido van Rossum (@gvanrossum) for creating the parser generator pgen2 + (originally used in lib2to3). + + + .. _jedi-vim: https://github.com/davidhalter/jedi-vim .. _youcompleteme: http://valloric.github.io/YouCompleteMe/ +.. _deoplete-jedi: https://github.com/zchee/deoplete-jedi .. _Jedi.el: https://github.com/tkf/emacs-jedi +.. _company-mode: https://github.com/syohex/emacs-company-jedi .. _elpy: https://github.com/jorgenschaefer/elpy .. _anaconda-mode: https://github.com/proofit404/anaconda-mode .. _ycmd: https://github.com/abingham/emacs-ycmd .. _sublimejedi: https://github.com/srusskih/SublimeJEDI .. _anaconda: https://github.com/DamnWidget/anaconda -.. _SynWrite: http://uvviewsoft.com/synjedi/ .. _wdb: https://github.com/Kozea/wdb .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle .. _Kate: http://kate-editor.org +.. _Atom: https://atom.io/ +.. _autocomplete-python: https://atom.io/packages/autocomplete-python +.. _SourceLair: https://www.sourcelair.com +.. _GNOME Builder: https://wiki.gnome.org/Apps/Builder +.. _Visual Studio Code: https://code.visualstudio.com/ +.. _gedi: https://github.com/isamert/gedi +.. _Eric IDE: http://eric-ide.python-projects.org diff --git a/docs/docs/features.rst b/docs/docs/features.rst index 48c41ead..fd5b3bab 100644 --- a/docs/docs/features.rst +++ b/docs/docs/features.rst @@ -20,12 +20,12 @@ make it work. General Features ---------------- -- python 2.6+ and 3.2+ support +- python 2.6+ and 3.3+ support - ignores syntax errors and wrong indentation - can deal with complex module / function / class structures - virtualenv support -- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings - (:ref:`type hinting `) +- can infer function arguments from sphinx, epydoc and basic numpydoc docstrings, + and PEP0484-style type hints (:ref:`type hinting `) Supported Python Features @@ -34,8 +34,8 @@ Supported Python Features |jedi| supports many of the widely used Python features: - builtins -- multiple returns or yields -- tuple assignments / array indexing / dictionary indexing +- returns, yields, yield from +- tuple assignments / array indexing / dictionary indexing / star unpacking - with-statement / exception handling - ``*args`` / ``**kwargs`` - decorators / lambdas / closures @@ -64,6 +64,7 @@ Not yet implemented: - manipulations of instances outside the instance variables without using methods +- implicit namespace packages (Python 3.3+, `PEP 420 `_) Will probably never be implemented: @@ -125,7 +126,49 @@ Type Hinting If |jedi| cannot detect the type of a function argument correctly (due to the dynamic nature of Python), you can help it by hinting the type using -one of the following docstring syntax styles: +one of the following docstring/annotation syntax styles: + +**PEP-0484 style** + +https://www.python.org/dev/peps/pep-0484/ + +function annotations (python 3 only; python 2 function annotations with +comments in planned but not yet implemented) + +:: + + def myfunction(node: ProgramNode, foo: str) -> None: + """Do something with a ``node``. + + """ + node.| # complete here + + +assignment, for-loop and with-statement type hints (all python versions). +Note that the type hints must be on the same line as the statement + +:: + + x = foo() # type: int + x, y = 2, 3 # type: typing.Optional[int], typing.Union[int, str] # typing module is mostly supported + for key, value in foo.items(): # type: str, Employee # note that Employee must be in scope + pass + with foo() as f: # type: int + print(f + 3) + +Most of the features in PEP-0484 are supported including the typing module +(for python < 3.5 you have to do ``pip install typing`` to use these), +and forward references. + +Things that are missing (and this is not an exhaustive list; some of these +are planned, others might be hard to implement and provide little worth): + +- annotating functions with comments: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code +- understanding ``typing.cast()`` +- stub files: https://www.python.org/dev/peps/pep-0484/#stub-files +- ``typing.Callable`` +- ``typing.TypeVar`` +- User defined generic types: https://www.python.org/dev/peps/pep-0484/#user-defined-generic-types **Sphinx style** diff --git a/docs/docs/plugin-api.rst b/docs/docs/plugin-api.rst index ce2c802d..93b081ef 100644 --- a/docs/docs/plugin-api.rst +++ b/docs/docs/plugin-api.rst @@ -47,14 +47,14 @@ Completions: >>> script = jedi.Script(source, 1, 19, '') >>> script - >>> completions = script.complete() + >>> completions = script.completions() >>> completions [, ] >>> completions[1] >>> completions[1].complete 'oads' - >>> completions[1].word + >>> completions[1].name 'loads' Definitions / Goto: diff --git a/docs/docs/static_analsysis.rst b/docs/docs/static_analsysis.rst new file mode 100644 index 00000000..7ad03c73 --- /dev/null +++ b/docs/docs/static_analsysis.rst @@ -0,0 +1,106 @@ + +This file is the start of the documentation of how static analysis works. + +Below is a list of parser names that are used within nodes_to_execute. + +------------ cared for: +global_stmt +exec_stmt # no priority +assert_stmt +if_stmt +while_stmt +for_stmt +try_stmt +(except_clause) +with_stmt +(with_item) +(with_var) +print_stmt +del_stmt +return_stmt +raise_stmt +yield_expr +file_input +funcdef +param +old_lambdef +lambdef +import_name +import_from +(import_as_name) +(dotted_as_name) +(import_as_names) +(dotted_as_names) +(dotted_name) +classdef +comp_for +(comp_if) ? +decorator + +----------- add basic +test +or_test +and_test +not_test +expr +xor_expr +and_expr +shift_expr +arith_expr +term +factor +power +atom +comparison +expr_stmt +testlist +testlist1 +testlist_safe + +----------- special care: +# mostly depends on how we handle the other ones. +testlist_star_expr # should probably just work with expr_stmt +star_expr +exprlist # just ignore? then names are just resolved. Strange anyway, bc expr is not really allowed in the list, typically. + +----------- ignore: +suite +subscriptlist +subscript +simple_stmt +?? sliceop # can probably just be added. +testlist_comp # prob ignore and care about it with atom. +dictorsetmaker +trailer +decorators +decorated +# always execute function arguments? -> no problem with stars. +# Also arglist and argument are different in different grammars. +arglist +argument + + +----------- remove: +tname # only exists in current Jedi parser. REMOVE! +tfpdef # python 2: tuple assignment; python 3: annotation +vfpdef # reduced in python 3 and therefore not existing. +tfplist # not in 3 +vfplist # not in 3 + +--------- not existing with parser reductions. +small_stmt +import_stmt +flow_stmt +compound_stmt +stmt +pass_stmt +break_stmt +continue_stmt +comp_op +augassign +old_test +typedargslist # afaik becomes [param] +varargslist # dito +vname +comp_iter +test_nocond diff --git a/docs/docs/usage.rst b/docs/docs/usage.rst index 960ca2ab..015c3e2a 100644 --- a/docs/docs/usage.rst +++ b/docs/docs/usage.rst @@ -21,6 +21,7 @@ Vim: - jedi-vim_ - YouCompleteMe_ +- deoplete-jedi_ Emacs: @@ -47,14 +48,33 @@ Kate: `__, you have to enable it, though. +Atom: -.. _other-software: +- autocomplete-python_ -Other Software Using Jedi -------------------------- +SourceLair: -- wdb_ - Web Debugger +- SourceLair_ +GNOME Builder: + +- `GNOME Builder`_ `supports it natively + `__, + and is enabled by default. + +Gedit: + +- gedi_ + +Eric IDE: + +- `Eric IDE`_ (Available as a plugin) + +Web Debugger: + +- wdb_ + +and many more! .. _repl-completion: @@ -77,6 +97,7 @@ Using a custom ``$HOME/.pythonrc.py`` .. _jedi-vim: https://github.com/davidhalter/jedi-vim .. _youcompleteme: http://valloric.github.io/YouCompleteMe/ +.. _deoplete-jedi: https://github.com/zchee/deoplete-jedi .. _Jedi.el: https://github.com/tkf/emacs-jedi .. _elpy: https://github.com/jorgenschaefer/elpy .. _anaconda-mode: https://github.com/proofit404/anaconda-mode @@ -86,3 +107,8 @@ Using a custom ``$HOME/.pythonrc.py`` .. _wdb: https://github.com/Kozea/wdb .. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle .. _kate: http://kate-editor.org/ +.. _autocomplete-python: https://atom.io/packages/autocomplete-python +.. _SourceLair: https://www.sourcelair.com +.. _GNOME Builder: https://wiki.gnome.org/Apps/Builder/ +.. _gedi: https://github.com/isamert/gedi +.. _Eric IDE: http://eric-ide.python-projects.org diff --git a/jedi/__main__.py b/jedi/__main__.py index b2639713..f2ee0477 100644 --- a/jedi/__main__.py +++ b/jedi/__main__.py @@ -1,18 +1,13 @@ -from sys import argv +import sys from os.path import join, dirname, abspath, isdir -if len(argv) == 2 and argv[1] == 'repl': - # don't want to use __main__ only for repl yet, maybe we want to use it for - # something else. So just use the keyword ``repl`` for now. - print(join(dirname(abspath(__file__)), 'api', 'replstartup.py')) -elif len(argv) > 1 and argv[1] == 'linter': +def _start_linter(): """ This is a pre-alpha API. You're not supposed to use it at all, except for testing. It will very likely change. """ import jedi - import sys if '--debug' in sys.argv: jedi.set_debug_function() @@ -37,7 +32,17 @@ elif len(argv) > 1 and argv[1] == 'linter': print(error) except Exception: if '--pdb' in sys.argv: + import traceback + traceback.print_exc() import pdb pdb.post_mortem() else: raise + + +if len(sys.argv) == 2 and sys.argv[1] == 'repl': + # don't want to use __main__ only for repl yet, maybe we want to use it for + # something else. So just use the keyword ``repl`` for now. + print(join(dirname(abspath(__file__)), 'api', 'replstartup.py')) +elif len(sys.argv) > 1 and sys.argv[1] == 'linter': + _start_linter() diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 60c9dcb5..0fde756c 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -13,6 +13,8 @@ except ImportError: is_py3 = sys.version_info[0] >= 3 is_py33 = is_py3 and sys.version_info.minor >= 3 +is_py34 = is_py3 and sys.version_info.minor >= 4 +is_py35 = is_py3 and sys.version_info.minor >= 5 is_py26 = not is_py3 and sys.version_info[1] < 7 @@ -25,7 +27,7 @@ def find_module_py33(string, path=None): except ValueError as e: # See #491. Importlib might raise a ValueError, to avoid this, we # just raise an ImportError to fix the issue. - raise ImportError("Originally ValueError: " + str(e)) + raise ImportError("Originally " + repr(e)) if loader is None: raise ImportError("Couldn't find a loader for {0}".format(string)) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 4e05f3e1..b9070493 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -3,38 +3,37 @@ The API basically only provides one class. You can create a :class:`Script` and use its methods. Additionally you can add a debug function with :func:`set_debug_function`. +Alternatively, if you don't need a custom function and are happy with printing +debug messages to stdout, simply call :func:`set_debug_function` without +arguments. .. warning:: Please, note that Jedi is **not thread safe**. """ -import re import os import warnings import sys -from itertools import chain -from jedi._compatibility import unicode, builtins -from jedi.parser import Parser, load_grammar -from jedi.parser.tokenize import source_tokens +from jedi._compatibility import unicode +from jedi.parser import load_grammar from jedi.parser import tree -from jedi.parser.user_context import UserContext, UserContextParser +from jedi.parser.fast import FastParser +from jedi.parser.utils import save_parser from jedi import debug from jedi import settings from jedi import common from jedi import cache -from jedi.api import keywords from jedi.api import classes from jedi.api import interpreter from jedi.api import usages from jedi.api import helpers +from jedi.api.completion import Completion from jedi.evaluate import Evaluator from jedi.evaluate import representation as er -from jedi.evaluate import compiled from jedi.evaluate import imports -from jedi.evaluate.cache import memoize_default -from jedi.evaluate.helpers import FakeName, get_module_names -from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names -from jedi.evaluate import analysis +from jedi.evaluate.param import try_iter_content +from jedi.evaluate.helpers import get_module_names from jedi.evaluate.sys_path import get_venv_path +from jedi.evaluate.iterable import unpack_tuple_to_dict # Jedi uses lots and lots of recursion. By setting this a little bit higher, we # can remove some "maximum recursion depth" errors. @@ -75,8 +74,8 @@ class Script(object): :type source: str :param line: The line to perform actions on (starting with 1). :type line: int - :param col: The column of the cursor (starting with 0). - :type col: int + :param column: The column of the cursor (starting with 0). + :type column: int :param path: The path of the file in the file system, or ``''`` if it hasn't been saved yet. :type path: str or None @@ -101,31 +100,30 @@ class Script(object): encoding = source_encoding self._orig_path = path - self.path = None if path is None else os.path.abspath(path) + # An empty path (also empty string) should always result in no path. + self.path = os.path.abspath(path) if path else None if source is None: + # TODO add a better warning than the traceback! with open(path) as f: source = f.read() - self.source = common.source_to_unicode(source, encoding) - lines = common.splitlines(self.source) - line = max(len(lines), 1) if line is None else line - if not (0 < line <= len(lines)): + self._source = common.source_to_unicode(source, encoding) + self._code_lines = common.splitlines(self._source) + line = max(len(self._code_lines), 1) if line is None else line + if not (0 < line <= len(self._code_lines)): raise ValueError('`line` parameter is not in a valid range.') - line_len = len(lines[line - 1]) + line_len = len(self._code_lines[line - 1]) column = line_len if column is None else column if not (0 <= column <= line_len): raise ValueError('`column` parameter is not in a valid range.') self._pos = line, column + self._path = path cache.clear_time_caches() debug.reset_time() - self._grammar = load_grammar('grammar%s.%s' % sys.version_info[:2]) - self._user_context = UserContext(self.source, self._pos) - self._parser = UserContextParser(self._grammar, self.source, path, - self._pos, self._user_context, - self._parsed_callback) + self._grammar = load_grammar(version='%s.%s' % sys.version_info[:2]) if sys_path is None: venv = os.getenv('VIRTUAL_ENV') if venv: @@ -133,9 +131,14 @@ class Script(object): self._evaluator = Evaluator(self._grammar, sys_path=sys_path) debug.speed('init') - def _parsed_callback(self, parser): + def _get_module(self): + cache.invalidate_star_import_cache(self._path) + parser = FastParser(self._grammar, self._source, self.path) + save_parser(self.path, parser, pickling=False) + module = self._evaluator.wrap(parser.module) imports.add_module(self._evaluator, unicode(module.name), module) + return parser.module @property def source_path(self): @@ -158,187 +161,14 @@ class Script(object): :return: Completion objects, sorted by name and __ comes last. :rtype: list of :class:`classes.Completion` """ - def get_completions(user_stmt, bs): - # TODO this closure is ugly. it also doesn't work with - # simple_complete (used for Interpreter), somehow redo. - module = self._evaluator.wrap(self._parser.module()) - names, level, only_modules, unfinished_dotted = \ - helpers.check_error_statements(module, self._pos) - completion_names = [] - if names is not None: - imp_names = tuple(str(n) for n in names if n.end_pos < self._pos) - i = imports.Importer(self._evaluator, imp_names, module, level) - completion_names = i.completion_names(self._evaluator, only_modules) - - # TODO this paragraph is necessary, but not sure it works. - context = self._user_context.get_context() - if not next(context).startswith('.'): # skip the path - if next(context) == 'from': - # completion is just "import" if before stands from .. - if unfinished_dotted: - return completion_names - else: - return keywords.keyword_names('import') - - if isinstance(user_stmt, tree.Import): - module = self._parser.module() - completion_names += imports.completion_names(self._evaluator, - user_stmt, self._pos) - return completion_names - - if names is None and not isinstance(user_stmt, tree.Import): - if not path and not dot: - # add keywords - completion_names += keywords.keyword_names(all=True) - # TODO delete? We should search for valid parser - # transformations. - completion_names += self._simple_complete(path, dot, like) - return completion_names - debug.speed('completions start') - path = self._user_context.get_path_until_cursor() - # Dots following an int are not the start of a completion but a float - # literal. - if re.search(r'^\d\.$', path): - return [] - path, dot, like = helpers.completion_parts(path) - - user_stmt = self._parser.user_stmt_with_whitespace() - - b = compiled.builtin - completion_names = get_completions(user_stmt, b) - - if not dot: - # add named params - for call_sig in self.call_signatures(): - # Allow protected access, because it's a public API. - module = call_sig._name.get_parent_until() - # Compiled modules typically don't allow keyword arguments. - if not isinstance(module, compiled.CompiledObject): - for p in call_sig.params: - # Allow access on _definition here, because it's a - # public API and we don't want to make the internal - # Name object public. - if p._definition.stars == 0: # no *args/**kwargs - completion_names.append(p._name) - - needs_dot = not dot and path - - comps = [] - comp_dct = {} - for c in set(completion_names): - n = str(c) - if settings.case_insensitive_completion \ - and n.lower().startswith(like.lower()) \ - or n.startswith(like): - if isinstance(c.parent, (tree.Function, tree.Class)): - # TODO I think this is a hack. It should be an - # er.Function/er.Class before that. - c = self._evaluator.wrap(c.parent).name - new = classes.Completion(self._evaluator, c, needs_dot, len(like)) - k = (new.name, new.complete) # key - if k in comp_dct and settings.no_completion_duplicates: - comp_dct[k]._same_name_completions.append(new) - else: - comp_dct[k] = new - comps.append(new) - + completion = Completion( + self._evaluator, self._get_module(), self._code_lines, + self._pos, self.call_signatures + ) + completions = completion.completions() debug.speed('completions end') - - return sorted(comps, key=lambda x: (x.name.startswith('__'), - x.name.startswith('_'), - x.name.lower())) - - def _simple_complete(self, path, dot, like): - if not path and not dot: - scope = self._parser.user_scope() - if not scope.is_scope(): # Might be a flow (if/while/etc). - scope = scope.get_parent_scope() - names_dicts = global_names_dict_generator( - self._evaluator, - self._evaluator.wrap(scope), - self._pos - ) - completion_names = [] - for names_dict, pos in names_dicts: - names = list(chain.from_iterable(names_dict.values())) - if not names: - continue - completion_names += filter_definition_names(names, self._parser.user_stmt(), pos) - elif self._get_under_cursor_stmt(path) is None: - return [] - else: - scopes = list(self._prepare_goto(path, True)) - completion_names = [] - debug.dbg('possible completion scopes: %s', scopes) - for s in scopes: - names = [] - for names_dict in s.names_dicts(search_global=False): - names += chain.from_iterable(names_dict.values()) - - completion_names += filter_definition_names(names, self._parser.user_stmt()) - return completion_names - - def _prepare_goto(self, goto_path, is_completion=False): - """ - Base for completions/goto. Basically it returns the resolved scopes - under cursor. - """ - debug.dbg('start: %s in %s', goto_path, self._parser.user_scope()) - - user_stmt = self._parser.user_stmt_with_whitespace() - if not user_stmt and len(goto_path.split('\n')) > 1: - # If the user_stmt is not defined and the goto_path is multi line, - # something's strange. Most probably the backwards tokenizer - # matched to much. - return [] - - if isinstance(user_stmt, tree.Import): - i, _ = helpers.get_on_import_stmt(self._evaluator, self._user_context, - user_stmt, is_completion) - if i is None: - return [] - scopes = [i] - else: - # just parse one statement, take it and evaluate it - eval_stmt = self._get_under_cursor_stmt(goto_path) - if eval_stmt is None: - return [] - - module = self._evaluator.wrap(self._parser.module()) - names, level, _, _ = helpers.check_error_statements(module, self._pos) - if names: - names = [str(n) for n in names] - i = imports.Importer(self._evaluator, names, module, level) - return i.follow() - - scopes = self._evaluator.eval_element(eval_stmt) - - return scopes - - @memoize_default() - def _get_under_cursor_stmt(self, cursor_txt, start_pos=None): - tokenizer = source_tokens(cursor_txt) - r = Parser(self._grammar, cursor_txt, tokenizer=tokenizer) - try: - # Take the last statement available that is not an endmarker. - # And because it's a simple_stmt, we need to get the first child. - stmt = r.module.children[-2].children[0] - except (AttributeError, IndexError): - return None - - user_stmt = self._parser.user_stmt() - if user_stmt is None: - # Set the start_pos to a pseudo position, that doesn't exist but - # works perfectly well (for both completions in docstrings and - # statements). - pos = start_pos or self._pos - else: - pos = user_stmt.start_pos - - stmt.move(pos[0] - 1, pos[1]) # Moving the offset. - stmt.parent = self._parser.user_scope() - return stmt + return completions def goto_definitions(self): """ @@ -352,39 +182,18 @@ class Script(object): :rtype: list of :class:`classes.Definition` """ - def resolve_import_paths(scopes): - for s in scopes.copy(): - if isinstance(s, imports.ImportWrapper): - scopes.remove(s) - scopes.update(resolve_import_paths(set(s.follow()))) - return scopes + leaf = self._get_module().name_for_position(self._pos) + if leaf is None: + leaf = self._get_module().get_leaf_for_position(self._pos) + if leaf is None: + return [] + definitions = helpers.evaluate_goto_definition(self._evaluator, leaf) - goto_path = self._user_context.get_path_under_cursor() - context = self._user_context.get_context() - definitions = set() - if next(context) in ('class', 'def'): - definitions = set([self._evaluator.wrap(self._parser.user_scope())]) - else: - # Fetch definition of callee, if there's no path otherwise. - if not goto_path: - definitions = set(signature._definition - for signature in self.call_signatures()) - - if re.match('\w[\w\d_]*$', goto_path) and not definitions: - user_stmt = self._parser.user_stmt() - if user_stmt is not None and user_stmt.type == 'expr_stmt': - for name in user_stmt.get_defined_names(): - if name.start_pos <= self._pos <= name.end_pos: - # TODO scaning for a name and then using it should be - # the default. - definitions = set(self._evaluator.goto_definition(name)) - - if not definitions and goto_path: - definitions = set(self._prepare_goto(goto_path)) - - definitions = resolve_import_paths(definitions) names = [s.name for s in definitions] defs = [classes.Definition(self._evaluator, name) for name in names] + # The additional set here allows the definitions to become unique in an + # API sense. In the internals we want to separate more things than in + # the API. return helpers.sorted_definitions(set(defs)) def goto_assignments(self): @@ -400,72 +209,14 @@ class Script(object): d = [classes.Definition(self._evaluator, d) for d in set(results)] return helpers.sorted_definitions(d) - def _goto(self, add_import_name=False): + def _goto(self): """ Used for goto_assignments and usages. - - :param add_import_name: Add the the name (if import) to the result. """ - def follow_inexistent_imports(defs): - """ Imports can be generated, e.g. following - `multiprocessing.dummy` generates an import dummy in the - multiprocessing module. The Import doesn't exist -> follow. - """ - definitions = set(defs) - for d in defs: - if isinstance(d.parent, tree.Import) \ - and d.start_pos == (0, 0): - i = imports.ImportWrapper(self._evaluator, d.parent).follow(is_goto=True) - definitions.remove(d) - definitions |= follow_inexistent_imports(i) - return definitions - - goto_path = self._user_context.get_path_under_cursor() - context = self._user_context.get_context() - user_stmt = self._parser.user_stmt() - user_scope = self._parser.user_scope() - - stmt = self._get_under_cursor_stmt(goto_path) - if stmt is None: + name = self._get_module().name_for_position(self._pos) + if name is None: return [] - - if user_scope is None: - last_name = None - else: - # Try to use the parser if possible. - last_name = user_scope.name_for_position(self._pos) - - if last_name is None: - last_name = stmt - while not isinstance(last_name, tree.Name): - try: - last_name = last_name.children[-1] - except AttributeError: - # Doesn't have a name in it. - return [] - - if next(context) in ('class', 'def'): - # The cursor is on a class/function name. - user_scope = self._parser.user_scope() - definitions = set([user_scope.name]) - elif isinstance(user_stmt, tree.Import): - s, name = helpers.get_on_import_stmt(self._evaluator, - self._user_context, user_stmt) - - definitions = self._evaluator.goto(name) - else: - # The Evaluator.goto function checks for definitions, but since we - # use a reverse tokenizer, we have new name_part objects, so we - # have to check the user_stmt here for positions. - if isinstance(user_stmt, tree.ExprStmt) \ - and isinstance(last_name.parent, tree.ExprStmt): - for name in user_stmt.get_defined_names(): - if name.start_pos <= self._pos <= name.end_pos: - return [name] - - defs = self._evaluator.goto(last_name) - definitions = follow_inexistent_imports(defs) - return definitions + return list(self._evaluator.goto(name)) def usages(self, additional_module_paths=()): """ @@ -481,8 +232,8 @@ class Script(object): temp, settings.dynamic_flow_information = \ settings.dynamic_flow_information, False try: - user_stmt = self._parser.user_stmt() - definitions = self._goto(add_import_name=True) + user_stmt = self._get_module().get_statement_for_position(self._pos) + definitions = self._goto() if not definitions and isinstance(user_stmt, tree.Import): # For not defined imports (goto doesn't find something, we take # the name as a definition. This is enough, because every name @@ -503,7 +254,7 @@ class Script(object): definitions) module = set([d.get_parent_until() for d in definitions]) - module.add(self._parser.module()) + module.add(self._get_module()) names = usages.usages(self._evaluator, definitions, module) for d in set(definitions): @@ -525,50 +276,59 @@ class Script(object): abs()# <-- cursor is here - This would return ``None``. + This would return an empty list.. :rtype: list of :class:`classes.CallSignature` """ - call_txt, call_index, key_name, start_pos = self._user_context.call_signature() - if call_txt is None: - return [] - - stmt = self._get_under_cursor_stmt(call_txt, start_pos) - if stmt is None: + call_signature_details = \ + helpers.get_call_signature_details(self._get_module(), self._pos) + if call_signature_details is None: return [] with common.scale_speed_settings(settings.scale_call_signatures): - origins = cache.cache_call_signatures(self._evaluator, stmt, - self.source, self._pos) + definitions = helpers.cache_call_signatures( + self._evaluator, + call_signature_details.bracket_leaf, + self._code_lines, + self._pos + ) debug.speed('func_call followed') - return [classes.CallSignature(self._evaluator, o.name, stmt, call_index, key_name) - for o in origins if hasattr(o, 'py__call__')] + return [classes.CallSignature(self._evaluator, d.name, + call_signature_details.bracket_leaf.start_pos, + call_signature_details.call_index, + call_signature_details.keyword_name_str) + for d in definitions if hasattr(d, 'py__call__')] def _analysis(self): - def check_types(types): - for typ in types: - try: - f = typ.iter_content - except AttributeError: - pass + self._evaluator.is_analysis = True + self._evaluator.analysis_modules = [self._get_module()] + try: + for node in self._get_module().nodes_to_execute(): + if node.type in ('funcdef', 'classdef'): + if node.type == 'classdef': + continue + raise NotImplementedError + er.Function(self._evaluator, node).get_decorated_func() + elif isinstance(node, tree.Import): + import_names = set(node.get_defined_names()) + if node.is_nested(): + import_names |= set(path[-1] for path in node.paths()) + for n in import_names: + imports.ImportWrapper(self._evaluator, n).follow() + elif node.type == 'expr_stmt': + types = self._evaluator.eval_element(node) + for testlist in node.children[:-1:2]: + # Iterate tuples. + unpack_tuple_to_dict(self._evaluator, types, testlist) else: - check_types(f()) + try_iter_content(self._evaluator.goto_definitions(node)) + self._evaluator.reset_recursion_limitations() - #statements = set(chain(*self._parser.module().used_names.values())) - nodes, imp_names, decorated_funcs = \ - analysis.get_module_statements(self._parser.module()) - # Sort the statements so that the results are reproducible. - for n in imp_names: - imports.ImportWrapper(self._evaluator, n).follow() - for node in sorted(nodes, key=lambda obj: obj.start_pos): - check_types(self._evaluator.eval_element(node)) - - for dec_func in decorated_funcs: - er.Function(self._evaluator, dec_func).get_decorated_func() - - ana = [a for a in self._evaluator.analysis if self.path == a.path] - return sorted(set(ana), key=lambda x: x.line) + ana = [a for a in self._evaluator.analysis if self.path == a.path] + return sorted(set(ana), key=lambda x: x.line) + finally: + self._evaluator.is_analysis = False class Interpreter(Script): @@ -582,7 +342,7 @@ class Interpreter(Script): >>> from os.path import join >>> namespace = locals() - >>> script = Interpreter('join().up', [namespace]) + >>> script = Interpreter('join("").up', [namespace]) >>> print(script.completions()[0].name) upper """ @@ -601,61 +361,19 @@ class Interpreter(Script): If `line` and `column` are None, they are assumed be at the end of `source`. """ - if type(namespaces) is not list or len(namespaces) == 0 or \ - any([type(x) is not dict for x in namespaces]): - raise TypeError("namespaces must be a non-empty list of dict") + try: + namespaces = [dict(n) for n in namespaces] + except Exception: + raise TypeError("namespaces must be a non-empty list of dicts.") super(Interpreter, self).__init__(source, **kwds) self.namespaces = namespaces - # Don't use the fast parser, because it does crazy stuff that we don't - # need in our very simple and small code here (that is always - # changing). - self._parser = UserContextParser(self._grammar, self.source, - self._orig_path, self._pos, - self._user_context, self._parsed_callback, - use_fast_parser=False) - interpreter.add_namespaces_to_parser(self._evaluator, namespaces, - self._parser.module()) + parser_module = super(Interpreter, self)._get_module() + self._module = interpreter.MixedModule(self._evaluator, parser_module, self.namespaces) - def _simple_complete(self, path, dot, like): - user_stmt = self._parser.user_stmt_with_whitespace() - is_simple_path = not path or re.search('^[\w][\w\d.]*$', path) - if isinstance(user_stmt, tree.Import) or not is_simple_path: - return super(Interpreter, self)._simple_complete(path, dot, like) - else: - class NamespaceModule(object): - def __getattr__(_, name): - for n in self.namespaces: - try: - return n[name] - except KeyError: - pass - raise AttributeError() - - def __dir__(_): - gen = (n.keys() for n in self.namespaces) - return list(set(chain.from_iterable(gen))) - - paths = path.split('.') if path else [] - - namespaces = (NamespaceModule(), builtins) - for p in paths: - old, namespaces = namespaces, [] - for n in old: - try: - namespaces.append(getattr(n, p)) - except Exception: - pass - - completion_names = [] - for namespace in namespaces: - for name in dir(namespace): - if name.lower().startswith(like.lower()): - scope = self._parser.module() - n = FakeName(name, scope) - completion_names.append(n) - return completion_names + def _get_module(self): + return self._module def defined_names(source, path=None, encoding='utf-8'): @@ -701,7 +419,7 @@ def names(source=None, path=None, encoding='utf-8', all_scopes=False, # Set line/column to a random position, because they don't matter. script = Script(source, line=1, column=0, path=path, encoding=encoding) defs = [classes.Definition(script._evaluator, name_part) - for name_part in get_module_names(script._parser.module(), all_scopes)] + for name_part in get_module_names(script._get_module(), all_scopes)] return sorted(filter(def_ref_filter, defs), key=lambda x: (x.line, x.column)) @@ -722,6 +440,8 @@ def set_debug_function(func_cb=debug.print_to_stdout, warnings=True, """ Define a callback debug function to get all the debug messages. + If you don't specify any arguments, debug messages will be printed to stdout. + :param func_cb: The callback function for debug messages, with n params. """ debug.debug_function = func_cb diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 75275dd2..9a2c9655 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -16,6 +16,7 @@ from jedi.evaluate import representation as er from jedi.evaluate import iterable from jedi.evaluate import imports from jedi.evaluate import compiled +from jedi.evaluate.compiled import mixed from jedi.api import keywords from jedi.evaluate.finder import filter_definition_names @@ -148,7 +149,7 @@ class BaseDefinition(object): if isinstance(stripped, er.InstanceElement): stripped = stripped.var - if isinstance(stripped, compiled.CompiledObject): + if isinstance(stripped, (compiled.CompiledObject, mixed.MixedObject)): return stripped.api_type() elif isinstance(stripped, iterable.Array): return 'instance' @@ -322,7 +323,7 @@ class BaseDefinition(object): elif self._definition.isinstance(tree.Import): return imports.ImportWrapper(self._evaluator, self._name).follow() else: - return [self._definition] + return set([self._definition]) @property @memoize_default() @@ -331,7 +332,7 @@ class BaseDefinition(object): Raises an ``AttributeError``if the definition is not callable. Otherwise returns a list of `Definition` that represents the params. """ - followed = self._follow_statements_imports() + followed = list(self._follow_statements_imports()) if not followed or not hasattr(followed[0], 'py__call__'): raise AttributeError() followed = followed[0] # only check the first one. @@ -365,33 +366,31 @@ class Completion(BaseDefinition): `Completion` objects are returned from :meth:`api.Script.completions`. They provide additional information about a completion. """ - def __init__(self, evaluator, name, needs_dot, like_name_length): + def __init__(self, evaluator, name, stack, like_name_length): super(Completion, self).__init__(evaluator, name) - self._needs_dot = needs_dot self._like_name_length = like_name_length + self._stack = stack # Completion objects with the same Completion name (which means # duplicate items in the completion) self._same_name_completions = [] def _complete(self, like_name): - dot = '.' if self._needs_dot else '' append = '' if settings.add_bracket_after_function \ and self.type == 'Function': append = '(' - if settings.add_dot_after_module: - if isinstance(self._definition, tree.Module): - append += '.' - if isinstance(self._definition, tree.Param): - append += '=' + if isinstance(self._definition, tree.Param) and self._stack is not None: + node_names = list(self._stack.get_node_names(self._evaluator.grammar)) + if 'trailer' in node_names and 'argument' not in node_names: + append += '=' name = str(self._name) if like_name: name = name[self._like_name_length:] - return dot + name + append + return name + append @property def complete(self): @@ -449,7 +448,7 @@ class Completion(BaseDefinition): followed = self._follow_statements_imports() if followed: # TODO: Use all of the followed objects as input to Documentation. - definition = followed[0] + definition = list(followed)[0] if raw: return _Help(definition).raw() @@ -629,11 +628,11 @@ class CallSignature(Definition): It knows what functions you are currently in. e.g. `isinstance(` would return the `isinstance` function. without `(` it would return nothing. """ - def __init__(self, evaluator, executable_name, call_stmt, index, key_name): + def __init__(self, evaluator, executable_name, bracket_start_pos, index, key_name_str): super(CallSignature, self).__init__(evaluator, executable_name) self._index = index - self._key_name = key_name - self._call_stmt = call_stmt + self._key_name_str = key_name_str + self._bracket_start_pos = bracket_start_pos @property def index(self): @@ -641,9 +640,9 @@ class CallSignature(Definition): The Param index of the current call. Returns None if the index cannot be found in the curent call. """ - if self._key_name is not None: + if self._key_name_str is not None: for i, param in enumerate(self.params): - if self._key_name == param.name: + if self._key_name_str == param.name: return i if self.params and self.params[-1]._name.get_definition().stars == 2: return i @@ -665,7 +664,7 @@ class CallSignature(Definition): The indent of the bracket that is responsible for the last function call. """ - return self._call_stmt.end_pos + return self._bracket_start_pos @property def call_name(self): diff --git a/jedi/api/completion.py b/jedi/api/completion.py new file mode 100644 index 00000000..a580fe9d --- /dev/null +++ b/jedi/api/completion.py @@ -0,0 +1,231 @@ +from itertools import chain + +from jedi.parser import token +from jedi.parser import tree +from jedi import debug +from jedi import settings +from jedi.api import classes +from jedi.api import helpers +from jedi.evaluate import imports +from jedi.api import keywords +from jedi.evaluate import compiled +from jedi.evaluate.helpers import call_of_leaf +from jedi.evaluate.finder import global_names_dict_generator, filter_definition_names + + +def get_call_signature_param_names(call_signatures): + # add named params + for call_sig in call_signatures: + # Allow protected access, because it's a public API. + module = call_sig._name.get_parent_until() + # Compiled modules typically don't allow keyword arguments. + if not isinstance(module, compiled.CompiledObject): + for p in call_sig.params: + # Allow access on _definition here, because it's a + # public API and we don't want to make the internal + # Name object public. + if p._definition.stars == 0: # no *args/**kwargs + yield p._name + + +def filter_names(evaluator, completion_names, stack, like_name): + comp_dct = {} + for name in set(completion_names): + if settings.case_insensitive_completion \ + and str(name).lower().startswith(like_name.lower()) \ + or str(name).startswith(like_name): + + if isinstance(name.parent, (tree.Function, tree.Class)): + # TODO I think this is a hack. It should be an + # er.Function/er.Class before that. + name = evaluator.wrap(name.parent).name + new = classes.Completion( + evaluator, + name, + stack, + len(like_name) + ) + k = (new.name, new.complete) # key + if k in comp_dct and settings.no_completion_duplicates: + comp_dct[k]._same_name_completions.append(new) + else: + comp_dct[k] = new + yield new + + +def get_user_scope(module, position): + """ + Returns the scope in which the user resides. This includes flows. + """ + user_stmt = module.get_statement_for_position(position) + if user_stmt is None: + def scan(scope): + for s in scope.children: + if s.start_pos <= position <= s.end_pos: + if isinstance(s, (tree.Scope, tree.Flow)): + return scan(s) or s + elif s.type in ('suite', 'decorated'): + return scan(s) + return None + + return scan(module) or module + else: + return user_stmt.get_parent_scope(include_flows=True) + + +class Completion: + def __init__(self, evaluator, module, code_lines, position, call_signatures_method): + self._evaluator = evaluator + self._module = evaluator.wrap(module) + self._code_lines = code_lines + + # The first step of completions is to get the name + self._like_name = helpers.get_on_completion_name(code_lines, position) + # The actual cursor position is not what we need to calculate + # everything. We want the start of the name we're on. + self._position = position[0], position[1] - len(self._like_name) + self._call_signatures_method = call_signatures_method + + def completions(self): + completion_names = self._get_context_completions() + + completions = filter_names(self._evaluator, completion_names, + self.stack, self._like_name) + + return sorted(completions, key=lambda x: (x.name.startswith('__'), + x.name.startswith('_'), + x.name.lower())) + + def _get_context_completions(self): + """ + Analyzes the context that a completion is made in and decides what to + return. + + Technically this works by generating a parser stack and analysing the + current stack for possible grammar nodes. + + Possible enhancements: + - global/nonlocal search global + - yield from / raise from <- could be only exceptions/generators + - In args: */**: no completion + - In params (also lambda): no completion before = + """ + + grammar = self._evaluator.grammar + + try: + self.stack = helpers.get_stack_at_position( + grammar, self._code_lines, self._module, self._position + ) + except helpers.OnErrorLeaf as e: + self.stack = None + if e.error_leaf.value == '.': + # After ErrorLeaf's that are dots, we will not do any + # completions since this probably just confuses the user. + return [] + # If we don't have a context, just use global completion. + + return self._global_completions() + + allowed_keywords, allowed_tokens = \ + helpers.get_possible_completion_types(grammar, self.stack) + + completion_names = list(self._get_keyword_completion_names(allowed_keywords)) + + if token.NAME in allowed_tokens: + # This means that we actually have to do type inference. + + symbol_names = list(self.stack.get_node_names(grammar)) + + nodes = list(self.stack.get_nodes()) + + if "import_stmt" in symbol_names: + level = 0 + only_modules = True + level, names = self._parse_dotted_names(nodes) + if "import_from" in symbol_names: + if 'import' in nodes: + only_modules = False + else: + assert "import_name" in symbol_names + + completion_names += self._get_importer_names( + names, + level, + only_modules + ) + elif nodes and nodes[-1] in ('as', 'def', 'class'): + # No completions for ``with x as foo`` and ``import x as foo``. + # Also true for defining names as a class or function. + return [] + elif symbol_names[-1] == 'trailer' and nodes[-1] == '.': + dot = self._module.get_leaf_for_position(self._position) + atom_expr = call_of_leaf(dot.get_previous_leaf()) + completion_names += self._trailer_completions(atom_expr) + else: + completion_names += self._global_completions() + + if 'trailer' in symbol_names: + call_signatures = self._call_signatures_method() + completion_names += get_call_signature_param_names(call_signatures) + + return completion_names + + def _get_keyword_completion_names(self, keywords_): + for k in keywords_: + yield keywords.keyword(self._evaluator, k).name + + def _global_completions(self): + scope = get_user_scope(self._module, self._position) + if not scope.is_scope(): # Might be a flow (if/while/etc). + scope = scope.get_parent_scope() + scope = self._evaluator.wrap(scope) + debug.dbg('global completion scope: %s', scope) + names_dicts = global_names_dict_generator( + self._evaluator, + scope, + self._position + ) + completion_names = [] + for names_dict, pos in names_dicts: + names = list(chain.from_iterable(names_dict.values())) + if not names: + continue + completion_names += filter_definition_names( + names, self._module.get_statement_for_position(self._position), pos + ) + return completion_names + + def _trailer_completions(self, atom_expr): + scopes = self._evaluator.eval_element(atom_expr) + completion_names = [] + debug.dbg('trailer completion scopes: %s', scopes) + for s in scopes: + names = [] + for names_dict in s.names_dicts(search_global=False): + names += chain.from_iterable(names_dict.values()) + + completion_names += filter_definition_names( + names, self._module.get_statement_for_position(self._position) + ) + return completion_names + + def _parse_dotted_names(self, nodes): + level = 0 + names = [] + for node in nodes[1:]: + if node in ('.', '...'): + if not names: + level += len(node.value) + elif node.type == 'dotted_name': + names += node.children[::2] + elif node.type == 'name': + names.append(node) + else: + break + return level, names + + def _get_importer_names(self, names, level=0, only_modules=True): + names = [str(n) for n in names] + i = imports.Importer(self._evaluator, names, self._module, level) + return i.completion_names(self._evaluator, only_modules=only_modules) diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index b1b3f6e4..a174eed6 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -2,18 +2,17 @@ Helpers for the API """ import re +from collections import namedtuple +from textwrap import dedent -from jedi.parser import tree as pt -from jedi.evaluate import imports +from jedi._compatibility import u +from jedi.evaluate.helpers import call_of_leaf +from jedi import parser +from jedi.parser import tokenize, token +from jedi.cache import time_cache -def completion_parts(path_until_cursor): - """ - Returns the parts for the completion - :return: tuple - (path, dot, like) - """ - match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path_until_cursor, flags=re.S) - return match.groups() +CompletionParts = namedtuple('CompletionParts', ['path', 'has_dot', 'name']) def sorted_definitions(defs): @@ -21,58 +20,254 @@ def sorted_definitions(defs): return sorted(defs, key=lambda x: (x.module_path or '', x.line or 0, x.column or 0)) -def get_on_import_stmt(evaluator, user_context, user_stmt, is_like_search=False): +def get_on_completion_name(lines, position): + line = lines[position[0] - 1] + # The first step of completions is to get the name + return re.search( + r'(?!\d)\w+$|$', line[:position[1]] + ).group(0) + + +def _get_code(code_lines, start_pos, end_pos): + # Get relevant lines. + lines = code_lines[start_pos[0] - 1:end_pos[0]] + # Remove the parts at the end of the line. + lines[-1] = lines[-1][:end_pos[1]] + # Remove first line indentation. + lines[0] = lines[0][start_pos[1]:] + return '\n'.join(lines) + + +class OnErrorLeaf(Exception): + @property + def error_leaf(self): + return self.args[0] + + +def get_stack_at_position(grammar, code_lines, module, pos): """ - Resolve the user statement, if it is an import. Only resolve the - parts until the user position. + Returns the possible node names (e.g. import_from, xor_test or yield_stmt). """ - name = user_stmt.name_for_position(user_context.position) - if name is None: - return None, None + user_stmt = module.get_statement_for_position(pos) - i = imports.ImportWrapper(evaluator, name) - return i, name + if user_stmt is not None and user_stmt.type in ('indent', 'dedent'): + code = u('') + else: + if user_stmt is None: + user_stmt = module.get_leaf_for_position(pos, include_prefixes=True) + if pos <= user_stmt.start_pos: + try: + leaf = user_stmt.get_previous_leaf() + except IndexError: + pass + else: + user_stmt = module.get_statement_for_position(leaf.start_pos) + + if user_stmt.type == 'error_leaf' or user_stmt.type == 'string': + # Error leafs cannot be parsed, completion in strings is also + # impossible. + raise OnErrorLeaf(user_stmt) + + start_pos = user_stmt.start_pos + if user_stmt.first_leaf() == '@': + # TODO this once again proves that just using user_stmt.get_code + # would probably be nicer than _get_code. + # Remove the indent to have a statement that is aligned (properties + # on the same line as function) + start_pos = start_pos[0], 0 + + code = _get_code(code_lines, start_pos, pos) + if code == ';': + # ; cannot be parsed. + code = u('') + + # Remove whitespace at the end. Necessary, because the tokenizer will parse + # an error token (there's no new line at the end in our case). This doesn't + # alter any truth about the valid tokens at that position. + code = code.rstrip('\t ') + # Remove as many indents from **all** code lines as possible. + code = dedent(code) + + class EndMarkerReached(Exception): + pass + + def tokenize_without_endmarker(code): + tokens = tokenize.source_tokens(code, use_exact_op_types=True) + for token_ in tokens: + if token_[0] == token.ENDMARKER: + raise EndMarkerReached() + elif token_[0] == token.DEDENT: + # Ignore those. Error statements should not contain them, if + # they do it's for cases where an indentation happens and + # before the endmarker we still see them. + pass + else: + yield token_ + + p = parser.Parser(grammar, code, start_parsing=False) + try: + p.parse(tokenizer=tokenize_without_endmarker(code)) + except EndMarkerReached: + return Stack(p.stack) -def check_error_statements(module, pos): - for error_statement in module.error_statement_stacks: - if error_statement.first_type in ('import_from', 'import_name') \ - and error_statement.first_pos < pos <= error_statement.next_start_pos: - return importer_from_error_statement(error_statement, pos) - return None, 0, False, False +class Stack(list): + def get_node_names(self, grammar): + for dfa, state, (node_number, nodes) in self: + yield grammar.number2symbol[node_number] - -def importer_from_error_statement(error_statement, pos): - def check_dotted(children): - for name in children[::2]: - if name.start_pos <= pos: - yield name - - names = [] - level = 0 - only_modules = True - unfinished_dotted = False - for typ, nodes in error_statement.stack: - if typ == 'dotted_name': - names += check_dotted(nodes) - if nodes[-1] == '.': - # An unfinished dotted_name - unfinished_dotted = True - elif typ == 'import_name': - if nodes[0].start_pos <= pos <= nodes[0].end_pos: - # We are on the import. - return None, 0, False, False - elif typ == 'import_from': + def get_nodes(self): + for dfa, state, (node_number, nodes) in self: for node in nodes: - if node.start_pos >= pos: - break - elif isinstance(node, pt.Node) and node.type == 'dotted_name': - names += check_dotted(node.children) - elif node in ('.', '...'): - level += len(node.value) - elif isinstance(node, pt.Name): - names.append(node) - elif node == 'import': - only_modules = False + yield node - return names, level, only_modules, unfinished_dotted + +def get_possible_completion_types(grammar, stack): + def add_results(label_index): + try: + grammar_labels.append(inversed_tokens[label_index]) + except KeyError: + try: + keywords.append(inversed_keywords[label_index]) + except KeyError: + t, v = grammar.labels[label_index] + assert t >= 256 + # See if it's a symbol and if we're in its first set + inversed_keywords + itsdfa = grammar.dfas[t] + itsstates, itsfirst = itsdfa + for first_label_index in itsfirst.keys(): + add_results(first_label_index) + + inversed_keywords = dict((v, k) for k, v in grammar.keywords.items()) + inversed_tokens = dict((v, k) for k, v in grammar.tokens.items()) + + keywords = [] + grammar_labels = [] + + def scan_stack(index): + dfa, state, node = stack[index] + states, first = dfa + arcs = states[state] + + for label_index, new_state in arcs: + if label_index == 0: + # An accepting state, check the stack below. + scan_stack(index - 1) + else: + add_results(label_index) + + scan_stack(-1) + + return keywords, grammar_labels + + +def evaluate_goto_definition(evaluator, leaf): + if leaf.type == 'name': + # In case of a name we can just use goto_definition which does all the + # magic itself. + return evaluator.goto_definitions(leaf) + + node = None + parent = leaf.parent + if parent.type == 'atom': + node = leaf.parent + elif parent.type == 'trailer': + node = call_of_leaf(leaf) + + if node is None: + return [] + return evaluator.eval_element(node) + + +CallSignatureDetails = namedtuple( + 'CallSignatureDetails', + ['bracket_leaf', 'call_index', 'keyword_name_str'] +) + + +def _get_index_and_key(nodes, position): + """ + Returns the amount of commas and the keyword argument string. + """ + nodes_before = [c for c in nodes if c.start_pos < position] + if nodes_before[-1].type == 'arglist': + nodes_before = [c for c in nodes_before[-1].children if c.start_pos < position] + + key_str = None + + if nodes_before: + last = nodes_before[-1] + if last.type == 'argument' and last.children[1].end_pos <= position: + # Checked if the argument + key_str = last.children[0].value + elif last == '=': + key_str = nodes_before[-2].value + + return nodes_before.count(','), key_str + + +def _get_call_signature_details_from_error_node(node, position): + for index, element in reversed(list(enumerate(node.children))): + # `index > 0` means that it's a trailer and not an atom. + if element == '(' and element.end_pos <= position and index > 0: + # It's an error node, we don't want to match too much, just + # until the parentheses is enough. + children = node.children[index:] + name = element.get_previous_leaf() + if name.type == 'name' or name.parent.type in ('trailer', 'atom'): + return CallSignatureDetails( + element, + *_get_index_and_key(children, position) + ) + + +def get_call_signature_details(module, position): + leaf = module.get_leaf_for_position(position, include_prefixes=True) + if leaf == ')': + if leaf.end_pos == position: + leaf = leaf.get_next_leaf() + # Now that we know where we are in the syntax tree, we start to look at + # parents for possible function definitions. + node = leaf.parent + while node is not None: + if node.type in ('funcdef', 'classdef'): + # Don't show call signatures if there's stuff before it that just + # makes it feel strange to have a call signature. + return None + + for n in node.children: + if n.start_pos < position and n.type == 'error_node': + result = _get_call_signature_details_from_error_node(n, position) + if result is not None: + return result + + if node.type == 'trailer' and node.children[0] == '(': + leaf = node.get_previous_leaf() + return CallSignatureDetails( + node.children[0], *_get_index_and_key(node.children, position)) + + node = node.parent + + return None + + +@time_cache("call_signatures_validity") +def cache_call_signatures(evaluator, bracket_leaf, code_lines, user_pos): + """This function calculates the cache key.""" + index = user_pos[0] - 1 + + before_cursor = code_lines[index][:user_pos[1]] + other_lines = code_lines[bracket_leaf.start_pos[0]:index] + whole = '\n'.join(other_lines + [before_cursor]) + before_bracket = re.match(r'.*\(', whole, re.DOTALL) + + module_path = bracket_leaf.get_parent_until().path + if module_path is None: + yield None # Don't cache! + else: + yield (module_path, before_bracket, bracket_leaf.start_pos) + yield evaluate_goto_definition( + evaluator, + bracket_leaf.get_previous_leaf() + ) diff --git a/jedi/api/interpreter.py b/jedi/api/interpreter.py index 595435c6..a6778a6c 100644 --- a/jedi/api/interpreter.py +++ b/jedi/api/interpreter.py @@ -1,30 +1,43 @@ """ TODO Some parts of this module are still not well documented. """ -import inspect -import re +import copy -from jedi._compatibility import builtins -from jedi import debug -from jedi.common import source_to_unicode from jedi.cache import underscore_memoization -from jedi.evaluate import compiled -from jedi.evaluate.compiled.fake import get_module -from jedi.parser import tree as pt -from jedi.parser import load_grammar -from jedi.parser.fast import FastParser from jedi.evaluate import helpers -from jedi.evaluate import iterable -from jedi.evaluate import representation as er +from jedi.evaluate.representation import ModuleWrapper +from jedi.evaluate.compiled import mixed -def add_namespaces_to_parser(evaluator, namespaces, parser_module): - for namespace in namespaces: - for key, value in namespace.items(): - # Name lookups in an ast tree work by checking names_dict. - # Therefore we just add fake names to that and we're done. - arr = parser_module.names_dict.setdefault(key, []) - arr.append(LazyName(evaluator, parser_module, key, value)) +class MixedModule(object): + resets_positions = True + type = 'mixed_module' + + def __init__(self, evaluator, parser_module, namespaces): + self._evaluator = evaluator + self._namespaces = namespaces + + self._namespace_objects = [type('jedi_namespace', (), n) for n in namespaces] + self._wrapped_module = ModuleWrapper(evaluator, parser_module) + # Usually we are dealing with very small code sizes when it comes to + # interpreter modules. In this case we just copy the whole syntax tree + # to be able to modify it. + self._parser_module = copy.deepcopy(parser_module) + + for child in self._parser_module.children: + child.parent = self + + def names_dicts(self, search_global): + for names_dict in self._wrapped_module.names_dicts(search_global): + yield names_dict + + for namespace_obj in self._namespace_objects: + m = mixed.MixedObject(self._evaluator, namespace_obj, self._parser_module.name) + for names_dict in m.names_dicts(False): + yield names_dict + + def __getattr__(self, name): + return getattr(self._parser_module, name) class LazyName(helpers.FakeName): @@ -43,66 +56,11 @@ class LazyName(helpers.FakeName): def parent(self): """ Creating fake statements for the interpreter. + + Here we are trying to link back to Python code, if possible. This means + we try to find the python module for a name (not the builtin). """ - obj = self._value - parser_path = [] - if inspect.ismodule(obj): - module = obj - else: - names = [] - try: - o = obj.__objclass__ - names.append(obj.__name__) - obj = o - except AttributeError: - pass - - try: - module_name = obj.__module__ - names.insert(0, obj.__name__) - except AttributeError: - # Unfortunately in some cases like `int` there's no __module__ - module = builtins - else: - # TODO this import is wrong. Yields x for x.y.z instead of z - module = __import__(module_name) - parser_path = names - raw_module = get_module(self._value) - - found = [] - try: - path = module.__file__ - except AttributeError: - pass - else: - path = re.sub('c$', '', path) - if path.endswith('.py'): - # cut the `c` from `.pyc` - with open(path) as f: - source = source_to_unicode(f.read()) - mod = FastParser(load_grammar(), source, path[:-1]).module - if parser_path: - assert len(parser_path) == 1 - found = self._evaluator.find_types(mod, parser_path[0], search_global=True) - else: - found = [self._evaluator.wrap(mod)] - - if not found: - debug.warning('Possibly an interpreter lookup for Python code failed %s', - parser_path) - - if not found: - evaluated = compiled.CompiledObject(obj) - if evaluated == builtins: - # The builtins module is special and always cached. - evaluated = compiled.builtin - found = [evaluated] - - content = iterable.AlreadyEvaluated(found) - stmt = pt.ExprStmt([self, pt.Operator(pt.zero_position_modifier, - '=', (0, 0), ''), content]) - stmt.parent = self._module - return stmt + return mixed.create(self._evaluator, self._value) @parent.setter def parent(self, value): diff --git a/jedi/api/keywords.py b/jedi/api/keywords.py index 2a54ba2d..365cb20c 100644 --- a/jedi/api/keywords.py +++ b/jedi/api/keywords.py @@ -1,11 +1,10 @@ import pydoc import keyword -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 from jedi import common -from jedi.evaluate import compiled from jedi.evaluate.helpers import FakeName - +from jedi.parser.tree import Leaf try: from pydoc_data import topics as pydoc_topics except ImportError: @@ -13,36 +12,75 @@ except ImportError: import pydoc_topics if is_py3: - keys = keyword.kwlist + if is_py35: + # in python 3.5 async and await are not proper keywords, but for + # completion pursposes should as as though they are + keys = keyword.kwlist + ["async", "await"] + else: + keys = keyword.kwlist else: keys = keyword.kwlist + ['None', 'False', 'True'] -def keywords(string='', pos=(0, 0), all=False): - if all: - return set([Keyword(k, pos) for k in keys]) +def has_inappropriate_leaf_keyword(pos, module): + relevant_errors = filter( + lambda error: error.first_pos[0] == pos[0], + module.error_statement_stacks) + + for error in relevant_errors: + if error.next_token in keys: + return True + + return False + + +def completion_names(evaluator, stmt, pos, module): + keyword_list = all_keywords(evaluator) + + if not isinstance(stmt, Leaf) or has_inappropriate_leaf_keyword(pos, module): + keyword_list = filter( + lambda keyword: not keyword.only_valid_as_leaf, + keyword_list + ) + return [keyword.name for keyword in keyword_list] + + +def all_keywords(evaluator, pos=(0, 0)): + return set([Keyword(evaluator, k, pos) for k in keys]) + + +def keyword(evaluator, string, pos=(0, 0)): if string in keys: - return set([Keyword(string, pos)]) - return set() + return Keyword(evaluator, string, pos) + else: + return None -def keyword_names(*args, **kwargs): - return [k.name for k in keywords(*args, **kwargs)] +def get_operator(evaluator, string, pos): + return Keyword(evaluator, string, pos) -def get_operator(string, pos): - return Keyword(string, pos) +keywords_only_valid_as_leaf = ( + 'continue', + 'break', +) class Keyword(object): - def __init__(self, name, pos): + type = 'completion_keyword' + + def __init__(self, evaluator, name, pos): self.name = FakeName(name, self, pos) self.start_pos = pos - self.parent = compiled.builtin + self.parent = evaluator.BUILTINS def get_parent_until(self): return self.parent + @property + def only_valid_as_leaf(self): + return self.name.value in keywords_only_valid_as_leaf + @property def names(self): """ For a `parsing.Name` like comparision """ diff --git a/jedi/cache.py b/jedi/cache.py index fe073352..5f618375 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -14,36 +14,13 @@ there are global variables, which are holding the cache information. Some of these variables are being cleaned after every API usage. """ import time -import os -import sys -import json -import hashlib -import gc -import inspect -import shutil -import re -try: - import cPickle as pickle -except ImportError: - import pickle from jedi import settings -from jedi import common -from jedi import debug +from jedi.parser.utils import parser_cache +from jedi.parser.utils import underscore_memoization _time_caches = {} -# for fast_parser, should not be deleted -parser_cache = {} - - -class ParserCacheItem(object): - def __init__(self, parser, change_time=None): - self.parser = parser - if change_time is None: - change_time = time.time() - self.change_time = change_time - def clear_time_caches(delete_all=False): """ Jedi caches many things, that should be completed after each completion @@ -70,11 +47,12 @@ def clear_time_caches(delete_all=False): def time_cache(time_add_setting): """ - s This decorator works as follows: Call it with a setting and after that use the function with a callable that returns the key. But: This function is only called if the key is not available. After a certain amount of time (`time_add_setting`) the cache is invalid. + + If the given key is None, the function will not be cached. """ def _temp(key_func): dct = {} @@ -99,56 +77,6 @@ def time_cache(time_add_setting): return _temp -@time_cache("call_signatures_validity") -def cache_call_signatures(evaluator, call, source, user_pos): - """This function calculates the cache key.""" - index = user_pos[0] - 1 - lines = common.splitlines(source) - - before_cursor = lines[index][:user_pos[1]] - other_lines = lines[call.start_pos[0]:index] - whole = '\n'.join(other_lines + [before_cursor]) - before_bracket = re.match(r'.*\(', whole, re.DOTALL) - - module_path = call.get_parent_until().path - yield None if module_path is None else (module_path, before_bracket, call.start_pos) - yield evaluator.eval_element(call) - - -def underscore_memoization(func): - """ - Decorator for methods:: - - class A(object): - def x(self): - if self._x: - self._x = 10 - return self._x - - Becomes:: - - class A(object): - @underscore_memoization - def x(self): - return 10 - - A now has an attribute ``_x`` written by this decorator. - """ - name = '_' + func.__name__ - - def wrapper(self): - try: - return getattr(self, name) - except AttributeError: - result = func(self) - if inspect.isgenerator(result): - result = list(result) - setattr(self, name, result) - return result - - return wrapper - - def memoize_method(method): """A normal memoize function.""" def wrapper(self, *args, **kwargs): @@ -192,6 +120,14 @@ def _invalidate_star_import_cache_module(module, only_main=False): else: del _time_caches['star_import_cache_validity'][module] + # This stuff was part of load_parser. However since we're most likely + # not going to use star import caching anymore, just ignore it. + #else: + # In case there is already a module cached and this module + # has to be reparsed, we also need to invalidate the import + # caches. + # _invalidate_star_import_cache_module(parser_cache_item.parser.module) + def invalidate_star_import_cache(path): """On success returns True.""" @@ -201,148 +137,3 @@ def invalidate_star_import_cache(path): pass else: _invalidate_star_import_cache_module(parser_cache_item.parser.module) - - -def load_parser(path): - """ - Returns the module or None, if it fails. - """ - p_time = os.path.getmtime(path) if path else None - try: - parser_cache_item = parser_cache[path] - if not path or p_time <= parser_cache_item.change_time: - return parser_cache_item.parser - else: - # In case there is already a module cached and this module - # has to be reparsed, we also need to invalidate the import - # caches. - _invalidate_star_import_cache_module(parser_cache_item.parser.module) - except KeyError: - if settings.use_filesystem_cache: - return ParserPickling.load_parser(path, p_time) - - -def save_parser(path, parser, pickling=True): - try: - p_time = None if path is None else os.path.getmtime(path) - except OSError: - p_time = None - pickling = False - - item = ParserCacheItem(parser, p_time) - parser_cache[path] = item - if settings.use_filesystem_cache and pickling: - ParserPickling.save_parser(path, item) - - -class ParserPickling(object): - - version = 24 - """ - Version number (integer) for file system cache. - - Increment this number when there are any incompatible changes in - parser representation classes. For example, the following changes - are regarded as incompatible. - - - Class name is changed. - - Class is moved to another module. - - Defined slot of the class is changed. - """ - - def __init__(self): - self.__index = None - self.py_tag = 'cpython-%s%s' % sys.version_info[:2] - """ - Short name for distinguish Python implementations and versions. - - It's like `sys.implementation.cache_tag` but for Python < 3.3 - we generate something similar. See: - http://docs.python.org/3/library/sys.html#sys.implementation - - .. todo:: Detect interpreter (e.g., PyPy). - """ - - def load_parser(self, path, original_changed_time): - try: - pickle_changed_time = self._index[path] - except KeyError: - return None - if original_changed_time is not None \ - and pickle_changed_time < original_changed_time: - # the pickle file is outdated - return None - - with open(self._get_hashed_path(path), 'rb') as f: - try: - gc.disable() - parser_cache_item = pickle.load(f) - finally: - gc.enable() - - debug.dbg('pickle loaded: %s', path) - parser_cache[path] = parser_cache_item - return parser_cache_item.parser - - def save_parser(self, path, parser_cache_item): - self.__index = None - try: - files = self._index - except KeyError: - files = {} - self._index = files - - with open(self._get_hashed_path(path), 'wb') as f: - pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL) - files[path] = parser_cache_item.change_time - - self._flush_index() - - @property - def _index(self): - if self.__index is None: - try: - with open(self._get_path('index.json')) as f: - data = json.load(f) - except (IOError, ValueError): - self.__index = {} - else: - # 0 means version is not defined (= always delete cache): - if data.get('version', 0) != self.version: - self.clear_cache() - self.__index = {} - else: - self.__index = data['index'] - return self.__index - - def _remove_old_modules(self): - # TODO use - change = False - if change: - self._flush_index(self) - self._index # reload index - - def _flush_index(self): - data = {'version': self.version, 'index': self._index} - with open(self._get_path('index.json'), 'w') as f: - json.dump(data, f) - self.__index = None - - def clear_cache(self): - shutil.rmtree(self._cache_directory()) - - def _get_hashed_path(self, path): - return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest()) - - def _get_path(self, file): - dir = self._cache_directory() - if not os.path.exists(dir): - os.makedirs(dir) - return os.path.join(dir, file) - - def _cache_directory(self): - return os.path.join(settings.cache_directory, self.py_tag) - - -# is a singleton -ParserPickling = ParserPickling() diff --git a/jedi/common.py b/jedi/common.py index 0e9afd11..3655c0bd 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -3,6 +3,7 @@ import sys import contextlib import functools import re +from itertools import chain from ast import literal_eval from jedi._compatibility import unicode, reraise @@ -134,7 +135,7 @@ def source_to_unicode(source, encoding=None): return possible_encoding.group(1) else: # the default if nothing else has been set -> PEP 263 - return encoding if encoding is not None else 'iso-8859-1' + return encoding if encoding is not None else 'utf-8' if isinstance(source, unicode): # only cast str/bytes @@ -152,3 +153,8 @@ def splitlines(string): Also different: Returns ``['']`` for an empty string input. """ return re.split('\n|\r\n', string) + + +def unite(iterable): + """Turns a two dimensional array into a one dimensional.""" + return set(chain.from_iterable(iterable)) diff --git a/jedi/debug.py b/jedi/debug.py index 5c59a7b6..e67cec27 100644 --- a/jedi/debug.py +++ b/jedi/debug.py @@ -3,23 +3,52 @@ import inspect import os import time +def _lazy_colorama_init(): + """ + Lazily init colorama if necessary, not to screw up stdout is debug not + enabled. + + This version of the function does nothing. + """ + pass + +_inited=False + try: if os.name == 'nt': - # does not work on Windows, as pyreadline and colorama interfere + # Does not work on Windows, as pyreadline and colorama interfere raise ImportError else: # Use colorama for nicer console output. from colorama import Fore, init from colorama import initialise - # pytest resets the stream at the end - causes troubles. Since after - # every output the stream is reset automatically we don't need this. - initialise.atexit_done = True - init() + def _lazy_colorama_init(): + """ + Lazily init colorama if necessary, not to screw up stdout is + debug not enabled. + + This version of the function does init colorama. + """ + global _inited + if not _inited: + # pytest resets the stream at the end - causes troubles. Since + # after every output the stream is reset automatically we don't + # need this. + initialise.atexit_done = True + try: + init() + except Exception: + # Colorama fails with initializing under vim and is buggy in + # version 0.3.6. + pass + _inited = True + except ImportError: class Fore(object): RED = '' GREEN = '' YELLOW = '' + MAGENTA = '' RESET = '' NOTICE = object() @@ -32,15 +61,15 @@ enable_notice = False # callback, interface: level, str debug_function = None -ignored_modules = ['jedi.evaluate.builtin', 'jedi.parser'] -_debug_indent = -1 +ignored_modules = ['jedi.parser'] +_debug_indent = 0 _start_time = time.time() def reset_time(): global _start_time, _debug_indent _start_time = time.time() - _debug_indent = -1 + _debug_indent = 0 def increase_indent(func): @@ -49,44 +78,53 @@ def increase_indent(func): global _debug_indent _debug_indent += 1 try: - result = func(*args, **kwargs) + return func(*args, **kwargs) finally: _debug_indent -= 1 - return result return wrapper -def dbg(message, *args): +def dbg(message, *args, **kwargs): """ Looks at the stack, to see if a debug message should be printed. """ + if kwargs: + # Python 2 compatibility, because it doesn't understand default args + # after *args. + color = kwargs.get('color') + if color is None: + raise TypeError("debug.dbg doesn't support more named arguments than color") + else: + color = 'GREEN' + if debug_function and enable_notice: frm = inspect.stack()[1] mod = inspect.getmodule(frm[0]) if not (mod.__name__ in ignored_modules): i = ' ' * _debug_indent - debug_function(NOTICE, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args)) + _lazy_colorama_init() + debug_function(color, i + 'dbg: ' + message % tuple(u(repr(a)) for a in args)) def warning(message, *args): if debug_function and enable_warning: i = ' ' * _debug_indent - debug_function(WARNING, i + 'warning: ' + message % tuple(u(repr(a)) for a in args)) + debug_function('RED', i + 'warning: ' + message % tuple(u(repr(a)) for a in args)) def speed(name): if debug_function and enable_speed: now = time.time() i = ' ' * _debug_indent - debug_function(SPEED, i + 'speed: ' + '%s %s' % (name, now - _start_time)) + debug_function('YELLOW', i + 'speed: ' + '%s %s' % (name, now - _start_time)) -def print_to_stdout(level, str_out): - """ The default debug function """ - if level == NOTICE: - col = Fore.GREEN - elif level == WARNING: - col = Fore.RED - else: - col = Fore.YELLOW +def print_to_stdout(color, str_out): + """ + The default debug function that prints to standard out. + + :param str color: A string that is an attribute of ``colorama.Fore``. + """ + col = getattr(Fore, color) + _lazy_colorama_init() if not is_py3: str_out = str_out.encode(encoding, 'replace') print(col + str_out + Fore.RESET) diff --git a/jedi/evaluate/__init__.py b/jedi/evaluate/__init__.py index 37c20fc1..0ca47aee 100644 --- a/jedi/evaluate/__init__.py +++ b/jedi/evaluate/__init__.py @@ -85,10 +85,13 @@ class Evaluator(object): self.memoize_cache = {} # for memoize decorators # To memorize modules -> equals `sys.modules`. self.modules = {} # like `sys.modules`. - self.compiled_cache = {} # see `compiled.create()` - self.recursion_detector = recursion.RecursionDetector() - self.execution_recursion_detector = recursion.ExecutionRecursionDetector() + self.compiled_cache = {} # see `evaluate.compiled.create()` + self.mixed_cache = {} # see `evaluate.compiled.mixed.create()` self.analysis = [] + self.predefined_if_name_dict_dict = {} + self.dynamic_params_depth = 0 + self.is_analysis = False + if sys_path is None: sys_path = sys.path self.sys_path = copy.copy(sys_path) @@ -97,16 +100,28 @@ class Evaluator(object): except ValueError: pass + self.reset_recursion_limitations() + + # Constants + self.BUILTINS = compiled.get_special_object(self, 'BUILTINS') + + def reset_recursion_limitations(self): + self.recursion_detector = recursion.RecursionDetector(self) + self.execution_recursion_detector = recursion.ExecutionRecursionDetector(self) + def wrap(self, element): - if isinstance(element, tree.Class): + if isinstance(element, (er.Wrapper, er.InstanceElement, + er.ModuleWrapper, er.FunctionExecution, er.Instance, compiled.CompiledObject)) or element is None: + # TODO this is so ugly, please refactor. + return element + + if element.type == 'classdef': return er.Class(self, element) - elif isinstance(element, tree.Function): - if isinstance(element, tree.Lambda): - return er.LambdaWrapper(self, element) - else: - return er.Function(self, element) - elif isinstance(element, (tree.Module)) \ - and not isinstance(element, er.ModuleWrapper): + elif element.type == 'funcdef': + return er.Function(self, element) + elif element.type == 'lambda': + return er.LambdaWrapper(self, element) + elif element.type == 'file_input': return er.ModuleWrapper(self, element) else: return element @@ -125,10 +140,10 @@ class Evaluator(object): scopes = f.scopes(search_global) if is_goto: return f.filter_name(scopes) - return f.find(scopes, search_global) + return f.find(scopes, attribute_lookup=not search_global) - @memoize_default(default=[], evaluator_is_first_arg=True) - @recursion.recursion_decorator + #@memoize_default(default=[], evaluator_is_first_arg=True) + #@recursion.recursion_decorator @debug.increase_indent def eval_statement(self, stmt, seek_name=None): """ @@ -140,10 +155,11 @@ class Evaluator(object): :param stmt: A `tree.ExprStmt`. """ debug.dbg('eval_statement %s (%s)', stmt, seek_name) - types = self.eval_element(stmt.get_rhs()) + rhs = stmt.get_rhs() + types = self.eval_element(rhs) if seek_name: - types = finder.check_tuple_assignments(types, seek_name) + types = finder.check_tuple_assignments(self, types, seek_name) first_operation = stmt.first_operation() if first_operation not in ('=', None) and not isinstance(stmt, er.InstanceElement): # TODO don't check for this. @@ -153,71 +169,168 @@ class Evaluator(object): name = str(stmt.get_defined_names()[0]) parent = self.wrap(stmt.get_parent_scope()) left = self.find_types(parent, name, stmt.start_pos, search_global=True) - if isinstance(stmt.get_parent_until(tree.ForStmt), tree.ForStmt): + + for_stmt = stmt.get_parent_until(tree.ForStmt) + if isinstance(for_stmt, tree.ForStmt) and types \ + and for_stmt.defines_one_name(): # Iterate through result and add the values, that's possible # only in for loops without clutter, because they are - # predictable. - for r in types: - left = precedence.calculate(self, left, operator, [r]) + # predictable. Also only do it, if the variable is not a tuple. + node = for_stmt.get_input_node() + for_iterables = self.eval_element(node) + ordered = list(iterable.py__iter__(self, for_iterables, node)) + + for index_types in ordered: + dct = {str(for_stmt.children[1]): index_types} + self.predefined_if_name_dict_dict[for_stmt] = dct + t = self.eval_element(rhs) + left = precedence.calculate(self, left, operator, t) types = left + if ordered: + # If there are no for entries, we cannot iterate and the + # types are defined by += entries. Therefore the for loop + # is never called. + del self.predefined_if_name_dict_dict[for_stmt] else: types = precedence.calculate(self, left, operator, types) debug.dbg('eval_statement result %s', types) return types - @memoize_default(evaluator_is_first_arg=True) def eval_element(self, element): if isinstance(element, iterable.AlreadyEvaluated): - return list(element) + return set(element) elif isinstance(element, iterable.MergedNodes): return iterable.unite(self.eval_element(e) for e in element) + if_stmt = element.get_parent_until((tree.IfStmt, tree.ForStmt, tree.IsScope)) + predefined_if_name_dict = self.predefined_if_name_dict_dict.get(if_stmt) + if predefined_if_name_dict is None and isinstance(if_stmt, tree.IfStmt): + if_stmt_test = if_stmt.children[1] + name_dicts = [{}] + # If we already did a check, we don't want to do it again -> If + # predefined_if_name_dict_dict is filled, we stop. + # We don't want to check the if stmt itself, it's just about + # the content. + if element.start_pos > if_stmt_test.end_pos: + # Now we need to check if the names in the if_stmt match the + # names in the suite. + if_names = helpers.get_names_of_node(if_stmt_test) + element_names = helpers.get_names_of_node(element) + str_element_names = [str(e) for e in element_names] + if any(str(i) in str_element_names for i in if_names): + for if_name in if_names: + definitions = self.goto_definitions(if_name) + # Every name that has multiple different definitions + # causes the complexity to rise. The complexity should + # never fall below 1. + if len(definitions) > 1: + if len(name_dicts) * len(definitions) > 16: + debug.dbg('Too many options for if branch evaluation %s.', if_stmt) + # There's only a certain amount of branches + # Jedi can evaluate, otherwise it will take to + # long. + name_dicts = [{}] + break + + original_name_dicts = list(name_dicts) + name_dicts = [] + for definition in definitions: + new_name_dicts = list(original_name_dicts) + for i, name_dict in enumerate(new_name_dicts): + new_name_dicts[i] = name_dict.copy() + new_name_dicts[i][str(if_name)] = [definition] + + name_dicts += new_name_dicts + else: + for name_dict in name_dicts: + name_dict[str(if_name)] = definitions + if len(name_dicts) > 1: + result = set() + for name_dict in name_dicts: + self.predefined_if_name_dict_dict[if_stmt] = name_dict + try: + result |= self._eval_element_not_cached(element) + finally: + del self.predefined_if_name_dict_dict[if_stmt] + return result + else: + return self._eval_element_if_evaluated(element) + return self._eval_element_cached(element) + else: + if predefined_if_name_dict: + return self._eval_element_not_cached(element) + else: + return self._eval_element_if_evaluated(element) + return self._eval_element_cached(element) + + def _eval_element_if_evaluated(self, element): + """ + TODO This function is temporary: Merge with eval_element. + """ + parent = element + while parent is not None: + parent = parent.parent + predefined_if_name_dict = self.predefined_if_name_dict_dict.get(parent) + if predefined_if_name_dict is not None: + return self._eval_element_not_cached(element) + return self._eval_element_cached(element) + + @memoize_default(evaluator_is_first_arg=True) + def _eval_element_cached(self, element): + return self._eval_element_not_cached(element) + + @debug.increase_indent + def _eval_element_not_cached(self, element): debug.dbg('eval_element %s@%s', element, element.start_pos) + types = set() if isinstance(element, (tree.Name, tree.Literal)) or tree.is_node(element, 'atom'): - return self._eval_atom(element) + types = self._eval_atom(element) elif isinstance(element, tree.Keyword): # For False/True/None if element.value in ('False', 'True', 'None'): - return [compiled.builtin.get_by_name(element.value)] - else: - return [] + types.add(compiled.builtin_from_name(self, element.value)) + # else: print e.g. could be evaluated like this in Python 2.7 elif element.isinstance(tree.Lambda): - return [er.LambdaWrapper(self, element)] + types = set([er.LambdaWrapper(self, element)]) elif element.isinstance(er.LambdaWrapper): - return [element] # TODO this is no real evaluation. + types = set([element]) # TODO this is no real evaluation. elif element.type == 'expr_stmt': - return self.eval_statement(element) - elif element.type == 'power': + types = self.eval_statement(element) + elif element.type in ('power', 'atom_expr'): types = self._eval_atom(element.children[0]) for trailer in element.children[1:]: if trailer == '**': # has a power operation. - raise NotImplementedError + right = self.eval_element(element.children[2]) + types = set(precedence.calculate(self, types, trailer, right)) + break types = self.eval_trailer(types, trailer) - - return types elif element.type in ('testlist_star_expr', 'testlist',): # The implicit tuple in statements. - return [iterable.ImplicitTuple(self, element)] + types = set([iterable.ImplicitTuple(self, element)]) elif element.type in ('not_test', 'factor'): types = self.eval_element(element.children[-1]) for operator in element.children[:-1]: - types = list(precedence.factor_calculate(self, types, operator)) - return types + types = set(precedence.factor_calculate(self, types, operator)) elif element.type == 'test': # `x if foo else y` case. - return (self.eval_element(element.children[0]) + - self.eval_element(element.children[-1])) + types = (self.eval_element(element.children[0]) | + self.eval_element(element.children[-1])) elif element.type == 'operator': # Must be an ellipsis, other operators are not evaluated. - return [] # Ignore for now. + assert element.value == '...' + types = set([compiled.create(self, Ellipsis)]) elif element.type == 'dotted_name': types = self._eval_atom(element.children[0]) for next_name in element.children[2::2]: - types = list(chain.from_iterable(self.find_types(typ, next_name) - for typ in types)) - return types + types = set(chain.from_iterable(self.find_types(typ, next_name) + for typ in types)) + types = types + elif element.type == 'eval_input': + types = self._eval_element_not_cached(element.children[0]) else: - return precedence.calculate_children(self, element.children) + types = precedence.calculate_children(self, element.children) + debug.dbg('eval_element result %s', types) + return types def _eval_atom(self, atom): """ @@ -229,6 +342,13 @@ class Evaluator(object): # This is the first global lookup. stmt = atom.get_definition() scope = stmt.get_parent_until(tree.IsScope, include_current=True) + if isinstance(scope, (tree.Function, er.FunctionExecution)): + # Adjust scope: If the name is not in the suite, it's a param + # default or annotation and will be resolved as part of the + # parent scope. + colon = scope.children.index(':') + if atom.start_pos < scope.children[colon + 1].start_pos: + scope = scope.get_parent_scope() if isinstance(stmt, tree.CompFor): stmt = stmt.get_parent_until((tree.ClassOrFunc, tree.ExprStmt)) if stmt.type != 'expr_stmt': @@ -237,42 +357,53 @@ class Evaluator(object): stmt = atom return self.find_types(scope, atom, stmt.start_pos, search_global=True) elif isinstance(atom, tree.Literal): - return [compiled.create(self, atom.eval())] + return set([compiled.create(self, atom.eval())]) else: c = atom.children + if c[0].type == 'string': + # Will be one string. + types = self._eval_atom(c[0]) + for string in c[1:]: + right = self._eval_atom(string) + types = precedence.calculate(self, types, '+', right) + return types # Parentheses without commas are not tuples. - if c[0] == '(' and not len(c) == 2 \ + elif c[0] == '(' and not len(c) == 2 \ and not(tree.is_node(c[1], 'testlist_comp') and len(c[1].children) > 1): return self.eval_element(c[1]) + try: comp_for = c[1].children[1] except (IndexError, AttributeError): pass else: - if isinstance(comp_for, tree.CompFor): - return [iterable.Comprehension.from_atom(self, atom)] - return [iterable.Array(self, atom)] + if comp_for == ':': + # Dict comprehensions have a colon at the 3rd index. + try: + comp_for = c[1].children[3] + except IndexError: + pass + + if comp_for.type == 'comp_for': + return set([iterable.Comprehension.from_atom(self, atom)]) + return set([iterable.Array(self, atom)]) def eval_trailer(self, types, trailer): trailer_op, node = trailer.children[:2] if node == ')': # `arglist` is optional. node = () - new_types = [] - for typ in types: - debug.dbg('eval_trailer: %s in scope %s', trailer, typ) - if trailer_op == '.': - new_types += self.find_types(typ, node) - elif trailer_op == '(': - new_types += self.execute(typ, node, trailer) - elif trailer_op == '[': - try: - get = typ.get_index_types - except AttributeError: - debug.warning("TypeError: '%s' object is not subscriptable" - % typ) - else: - new_types += get(self, node) + + new_types = set() + if trailer_op == '[': + new_types |= iterable.py__getitem__(self, types, trailer) + else: + for typ in types: + debug.dbg('eval_trailer: %s in scope %s', trailer, typ) + if trailer_op == '.': + new_types |= self.find_types(typ, node) + elif trailer_op == '(': + new_types |= self.execute(typ, node, trailer) return new_types def execute_evaluated(self, obj, *args): @@ -287,6 +418,9 @@ class Evaluator(object): if not isinstance(arguments, param.Arguments): arguments = param.Arguments(self, arguments, trailer) + if self.is_analysis: + arguments.eval_all() + if obj.isinstance(er.Function): obj = obj.get_decorated_func() @@ -302,17 +436,28 @@ class Evaluator(object): func = obj.py__call__ except AttributeError: debug.warning("no execution possible %s", obj) - return [] + return set() else: - types = func(self, arguments) + types = func(arguments) debug.dbg('execute result: %s in %s', types, obj) return types - def goto_definition(self, name): + def goto_definitions(self, name): def_ = name.get_definition() - if def_.type == 'expr_stmt' and name in def_.get_defined_names(): - return self.eval_statement(def_, name) - call = helpers.call_of_name(name) + is_simple_name = name.parent.type not in ('power', 'trailer') + if is_simple_name: + if name.parent.type in ('file_input', 'classdef', 'funcdef'): + return [self.wrap(name.parent)] + if def_.type == 'expr_stmt' and name in def_.get_defined_names(): + return self.eval_statement(def_, name) + elif def_.type == 'for_stmt': + container_types = self.eval_element(def_.children[3]) + for_types = iterable.py__iter__types(self, container_types, def_.children[3]) + return finder.check_tuple_assignments(self, for_types, name) + elif def_.type in ('import_from', 'import_name'): + return imports.ImportWrapper(self, name).follow() + + call = helpers.call_of_leaf(name) return self.eval_element(call) def goto(self, name): @@ -372,8 +517,8 @@ class Evaluator(object): )) scope = name.get_parent_scope() - if tree.is_node(name.parent, 'trailer'): - call = helpers.call_of_name(name, cut_own_trailer=True) + if tree.is_node(par, 'trailer') and par.children[0] == '.': + call = helpers.call_of_leaf(name, cut_own_trailer=True) types = self.eval_element(call) return resolve_implicit_imports(iterable.unite( self.find_types(typ, name, is_goto=True) for typ in types diff --git a/jedi/evaluate/analysis.py b/jedi/evaluate/analysis.py index d4a411f4..7b4b0acc 100644 --- a/jedi/evaluate/analysis.py +++ b/jedi/evaluate/analysis.py @@ -5,19 +5,25 @@ from jedi import debug from jedi.parser import tree from jedi.evaluate.compiled import CompiledObject +from jedi.common import unite + CODES = { 'attribute-error': (1, AttributeError, 'Potential AttributeError.'), 'name-error': (2, NameError, 'Potential NameError.'), 'import-error': (3, ImportError, 'Potential ImportError.'), - 'type-error-generator': (4, TypeError, "TypeError: 'generator' object is not subscriptable."), - 'type-error-too-many-arguments': (5, TypeError, None), - 'type-error-too-few-arguments': (6, TypeError, None), - 'type-error-keyword-argument': (7, TypeError, None), - 'type-error-multiple-values': (8, TypeError, None), - 'type-error-star-star': (9, TypeError, None), - 'type-error-star': (10, TypeError, None), - 'type-error-operation': (11, TypeError, None), + 'type-error-too-many-arguments': (4, TypeError, None), + 'type-error-too-few-arguments': (5, TypeError, None), + 'type-error-keyword-argument': (6, TypeError, None), + 'type-error-multiple-values': (7, TypeError, None), + 'type-error-star-star': (8, TypeError, None), + 'type-error-star': (9, TypeError, None), + 'type-error-operation': (10, TypeError, None), + 'type-error-not-iterable': (11, TypeError, None), + 'type-error-isinstance': (12, TypeError, None), + 'type-error-not-subscriptable': (13, TypeError, None), + 'value-error-too-many-values': (14, ValueError, None), + 'value-error-too-few-values': (15, ValueError, None), } @@ -158,8 +164,8 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): from jedi.evaluate import iterable if isinstance(cls, iterable.Array) and cls.type == 'tuple': # multiple exceptions - for c in cls.values(): - if check_match(c, exception): + for typ in unite(cls.py__iter__()): + if check_match(typ, exception): return True else: if check_match(cls, exception): @@ -168,7 +174,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): def check_hasattr(node, suite): try: assert suite.start_pos <= jedi_obj.start_pos < suite.end_pos - assert node.type == 'power' + assert node.type in ('power', 'atom_expr') base = node.children[0] assert base.type == 'name' and base.value == 'hasattr' trailer = node.children[1] @@ -183,7 +189,7 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): # Check name key, values = args[1] assert len(values) == 1 - names = evaluator.eval_element(values[0]) + names = list(evaluator.eval_element(values[0])) assert len(names) == 1 and isinstance(names[0], CompiledObject) assert names[0].obj == str(payload[1]) @@ -208,95 +214,3 @@ def _check_for_exception_catch(evaluator, jedi_obj, exception, payload=None): obj = obj.parent return False - - -def get_module_statements(module): - """ - Returns the statements used in a module. All these statements should be - evaluated to check for potential exceptions. - """ - def check_children(node): - try: - children = node.children - except AttributeError: - return [] - else: - nodes = [] - for child in children: - nodes += check_children(child) - if child.type == 'trailer': - c = child.children - if c[0] == '(' and c[1] != ')': - if c[1].type != 'arglist': - if c[1].type == 'argument': - nodes.append(c[1].children[-1]) - else: - nodes.append(c[1]) - else: - for argument in c[1].children: - if argument.type == 'argument': - nodes.append(argument.children[-1]) - elif argument.type != 'operator': - nodes.append(argument) - return nodes - - def add_nodes(nodes): - new = set() - for node in nodes: - if isinstance(node, tree.Flow): - children = node.children - if node.type == 'for_stmt': - children = children[2:] # Don't want to include the names. - # Pick the suite/simple_stmt. - new |= add_nodes(children) - elif node.type in ('simple_stmt', 'suite'): - new |= add_nodes(node.children) - elif node.type in ('return_stmt', 'yield_expr'): - try: - new.add(node.children[1]) - except IndexError: - pass - elif node.type not in ('whitespace', 'operator', 'keyword', - 'parameters', 'decorated', 'except_clause') \ - and not isinstance(node, (tree.ClassOrFunc, tree.Import)): - new.add(node) - - try: - children = node.children - except AttributeError: - pass - else: - for next_node in children: - new.update(check_children(node)) - if next_node.type != 'keyword' and node.type != 'expr_stmt': - new.add(node) - return new - - nodes = set() - import_names = set() - decorated_funcs = [] - for scope in module.walk(): - for imp in set(scope.imports): - import_names |= set(imp.get_defined_names()) - if imp.is_nested(): - import_names |= set(path[-1] for path in imp.paths()) - - children = scope.children - if isinstance(scope, tree.ClassOrFunc): - children = children[2:] # We don't want to include the class name. - nodes |= add_nodes(children) - - for flow in scope.flows: - if flow.type == 'for_stmt': - nodes.add(flow.children[3]) - elif flow.type == 'try_stmt': - nodes.update(e for e in flow.except_clauses() if e is not None) - - try: - decorators = scope.get_decorators() - except AttributeError: - pass - else: - if decorators: - decorated_funcs.append(scope) - return nodes, import_names, decorated_funcs diff --git a/jedi/evaluate/compiled/__init__.py b/jedi/evaluate/compiled/__init__.py index 7224067f..7d451e6d 100644 --- a/jedi/evaluate/compiled/__init__.py +++ b/jedi/evaluate/compiled/__init__.py @@ -41,34 +41,30 @@ class CompiledObject(Base): path = None # modules have this attribute - set it to None. used_names = {} # To be consistent with modules. - def __init__(self, obj, parent=None): + def __init__(self, evaluator, obj, parent=None): + self._evaluator = evaluator self.obj = obj self.parent = parent - @property - def py__call__(self): - def actual(evaluator, params): - if inspect.isclass(self.obj): - from jedi.evaluate.representation import Instance - return [Instance(evaluator, self, params)] - else: - return list(self._execute_function(evaluator, params)) - - # Might raise an AttributeError, which is intentional. - self.obj.__call__ - return actual + @CheckAttribute + def py__call__(self, params): + if inspect.isclass(self.obj): + from jedi.evaluate.representation import Instance + return set([Instance(self._evaluator, self, params)]) + else: + return set(self._execute_function(params)) @CheckAttribute - def py__class__(self, evaluator): - return CompiledObject(self.obj.__class__, parent=self.parent) + def py__class__(self): + return create(self._evaluator, self.obj.__class__) @CheckAttribute - def py__mro__(self, evaluator): - return tuple(create(evaluator, cls, self.parent) for cls in self.obj.__mro__) + def py__mro__(self): + return tuple(create(self._evaluator, cls) for cls in self.obj.__mro__) @CheckAttribute - def py__bases__(self, evaluator): - return tuple(create(evaluator, cls) for cls in self.obj.__bases__) + def py__bases__(self): + return tuple(create(self._evaluator, cls) for cls in self.obj.__bases__) def py__bool__(self): return bool(self.obj) @@ -87,7 +83,7 @@ class CompiledObject(Base): def params(self): params_str, ret = self._parse_function_doc() tokens = params_str.split(',') - if inspect.ismethoddescriptor(self._cls().obj): + if inspect.ismethoddescriptor(self.obj): tokens.insert(0, 'self') params = [] for p in tokens: @@ -108,22 +104,21 @@ class CompiledObject(Base): return _parse_function_doc(self.doc) def api_type(self): - if fake.is_class_instance(self.obj): - return 'instance' - - cls = self._cls().obj - if inspect.isclass(cls): + obj = self.obj + if inspect.isclass(obj): return 'class' - elif inspect.ismodule(cls): + elif inspect.ismodule(obj): return 'module' - elif inspect.isbuiltin(cls) or inspect.ismethod(cls) \ - or inspect.ismethoddescriptor(cls): + elif inspect.isbuiltin(obj) or inspect.ismethod(obj) \ + or inspect.ismethoddescriptor(obj) or inspect.isfunction(obj): return 'function' + # Everything else... + return 'instance' @property def type(self): """Imitate the tree.Node.type values.""" - cls = self._cls().obj + cls = self._get_class() if inspect.isclass(cls): return 'classdef' elif inspect.ismodule(cls): @@ -134,17 +129,24 @@ class CompiledObject(Base): @underscore_memoization def _cls(self): + """ + We used to limit the lookups for instantiated objects like list(), but + this is not the case anymore. Python itself + """ # Ensures that a CompiledObject is returned that is not an instance (like list) - if fake.is_class_instance(self.obj): - try: - c = self.obj.__class__ - except AttributeError: - # happens with numpy.core.umath._UFUNC_API (you get it - # automatically by doing `import numpy`. - c = type(None) - return CompiledObject(c, self.parent) return self + def _get_class(self): + if not fake.is_class_instance(self.obj): + return self.obj + + try: + return self.obj.__class__ + except AttributeError: + # happens with numpy.core.umath._UFUNC_API (you get it + # automatically by doing `import numpy`. + return type + @property def names_dict(self): # For compatibility with `representation.Class`. @@ -159,64 +161,55 @@ class CompiledObject(Base): search_global shouldn't change the fact that there's one dict, this way there's only one `object`. """ - return [LazyNamesDict(self._cls(), is_instance)] + return [LazyNamesDict(self._evaluator, self, is_instance)] def get_subscope_by_name(self, name): - if name in dir(self._cls().obj): - return CompiledName(self._cls(), name).parent + if name in dir(self.obj): + return CompiledName(self._evaluator, self, name).parent else: raise KeyError("CompiledObject doesn't have an attribute '%s'." % name) - def get_index_types(self, evaluator, index_array=()): - # If the object doesn't have `__getitem__`, just raise the - # AttributeError. - if not hasattr(self.obj, '__getitem__'): - debug.warning('Tried to call __getitem__ on non-iterable.') - return [] + @CheckAttribute + def py__getitem__(self, index): if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict): # Get rid of side effects, we won't call custom `__getitem__`s. - return [] + return set() - result = [] - from jedi.evaluate.iterable import create_indexes_or_slices - for typ in create_indexes_or_slices(evaluator, index_array): - index = None - try: - index = typ.obj - new = self.obj[index] - except (KeyError, IndexError, TypeError, AttributeError): - # Just try, we don't care if it fails, except for slices. - if isinstance(index, slice): - result.append(self) - else: - result.append(CompiledObject(new)) - if not result: - try: - for obj in self.obj: - result.append(CompiledObject(obj)) - except TypeError: - pass # self.obj maynot have an __iter__ method. - return result + return set([create(self._evaluator, self.obj[index])]) + + @CheckAttribute + def py__iter__(self): + if type(self.obj) not in (str, list, tuple, unicode, bytes, bytearray, dict): + # Get rid of side effects, we won't call custom `__getitem__`s. + return + + for part in self.obj: + yield set([create(self._evaluator, part)]) @property def name(self): - # might not exist sometimes (raises AttributeError) - return FakeName(self._cls().obj.__name__, self) + try: + name = self._get_class().__name__ + except AttributeError: + name = repr(self.obj) + return FakeName(name, self) - def _execute_function(self, evaluator, params): + def _execute_function(self, params): if self.type != 'funcdef': return for name in self._parse_function_doc()[1].split(): try: - bltn_obj = _create_from_name(builtin, builtin, name) + bltn_obj = getattr(_builtins, name) except AttributeError: continue else: - if isinstance(bltn_obj, CompiledObject) and bltn_obj.obj is None: - # We want everything except None. + if bltn_obj is None: + # We want to evaluate everything except None. + # TODO do we? continue - for result in evaluator.execute(bltn_obj, params): + bltn_obj = create(self._evaluator, bltn_obj) + for result in self._evaluator.execute(bltn_obj, params): yield result @property @@ -228,7 +221,7 @@ class CompiledObject(Base): """ module = self.get_parent_until() faked_subscopes = [] - for name in dir(self._cls().obj): + for name in dir(self.obj): f = fake.get_faked(module.obj, self.obj, name) if f: f.parent = self @@ -245,11 +238,42 @@ class CompiledObject(Base): return [] # Builtins don't have imports +class CompiledName(FakeName): + def __init__(self, evaluator, compiled_obj, name): + super(CompiledName, self).__init__(name) + self._evaluator = evaluator + self._compiled_obj = compiled_obj + self.name = name + + def __repr__(self): + try: + name = self._compiled_obj.name # __name__ is not defined all the time + except AttributeError: + name = None + return '<%s: (%s).%s>' % (type(self).__name__, name, self.name) + + def is_definition(self): + return True + + @property + @underscore_memoization + def parent(self): + module = self._compiled_obj.get_parent_until() + return _create_from_name(self._evaluator, module, self._compiled_obj, self.name) + + @parent.setter + def parent(self, value): + pass # Just ignore this, FakeName tries to overwrite the parent attribute. + + class LazyNamesDict(object): """ A names_dict instance for compiled objects, resembles the parser.tree. """ - def __init__(self, compiled_obj, is_instance): + name_class = CompiledName + + def __init__(self, evaluator, compiled_obj, is_instance=False): + self._evaluator = evaluator self._compiled_obj = compiled_obj self._is_instance = is_instance @@ -262,7 +286,12 @@ class LazyNamesDict(object): getattr(self._compiled_obj.obj, name) except AttributeError: raise KeyError('%s in %s not found.' % (name, self._compiled_obj)) - return [CompiledName(self._compiled_obj, name)] + except Exception: + # This is a bit ugly. We're basically returning this to make + # lookups possible without having the actual attribute. However + # this makes proper completion possible. + return [FakeName(name, create(self._evaluator, None), is_definition=True)] + return [self.name_class(self._evaluator, self._compiled_obj, name)] def values(self): obj = self._compiled_obj.obj @@ -275,39 +304,13 @@ class LazyNamesDict(object): # The dir function can be wrong. pass - # dir doesn't include the type names. - if not inspect.ismodule(obj) and obj != type and not self._is_instance: - values += _type_names_dict.values() + is_instance = self._is_instance or fake.is_class_instance(obj) + # ``dir`` doesn't include the type names. + if not inspect.ismodule(obj) and obj != type and not is_instance: + values += create(self._evaluator, type).names_dict.values() return values -class CompiledName(FakeName): - def __init__(self, obj, name): - super(CompiledName, self).__init__(name) - self._obj = obj - self.name = name - - def __repr__(self): - try: - name = self._obj.name # __name__ is not defined all the time - except AttributeError: - name = None - return '<%s: (%s).%s>' % (type(self).__name__, name, self.name) - - def is_definition(self): - return True - - @property - @underscore_memoization - def parent(self): - module = self._obj.get_parent_until() - return _create_from_name(module, self._obj, self.name) - - @parent.setter - def parent(self, value): - pass # Just ignore this, FakeName tries to overwrite the parent attribute. - - def dotted_from_fs_path(fs_path, sys_path): """ Changes `/usr/lib/python3.4/email/utils.py` to `email.utils`. I.e. @@ -369,7 +372,7 @@ def load_module(evaluator, path=None, name=None): # complicated import structure of Python. module = sys.modules[dotted_path] - return CompiledObject(module) + return create(evaluator, module) docstr_defaults = { @@ -441,19 +444,7 @@ def _parse_function_doc(doc): return param_str, ret -class Builtin(CompiledObject): - @memoize_method - def get_by_name(self, name): - return self.names_dict[name][0].parent - - -def _a_generator(foo): - """Used to have an object to return for generators.""" - yield 42 - yield foo - - -def _create_from_name(module, parent, name): +def _create_from_name(evaluator, module, parent, name): faked = fake.get_faked(module.obj, parent.obj, name) # only functions are necessary. if faked is not None: @@ -467,61 +458,78 @@ def _create_from_name(module, parent, name): # PyQt4.QtGui.QStyleOptionComboBox.currentText # -> just set it to None obj = None - return CompiledObject(obj, parent) + return create(evaluator, obj, parent) -builtin = Builtin(_builtins) -magic_function_class = CompiledObject(type(load_module), parent=builtin) -generator_obj = CompiledObject(_a_generator(1.0)) -_type_names_dict = builtin.get_by_name('type').names_dict -none_obj = builtin.get_by_name('None') -false_obj = builtin.get_by_name('False') -true_obj = builtin.get_by_name('True') -object_obj = builtin.get_by_name('object') +def builtin_from_name(evaluator, string): + bltn_obj = getattr(_builtins, string) + return create(evaluator, bltn_obj) -def keyword_from_value(obj): - if obj is None: - return none_obj - elif obj is False: - return false_obj - elif obj is True: - return true_obj - else: - raise NotImplementedError +def _a_generator(foo): + """Used to have an object to return for generators.""" + yield 42 + yield foo -def compiled_objects_cache(func): - def wrapper(evaluator, obj, parent=builtin, module=None): - # Do a very cheap form of caching here. - key = id(obj), id(parent), id(module) - try: - return evaluator.compiled_cache[key][0] - except KeyError: - result = func(evaluator, obj, parent, module) - # Need to cache all of them, otherwise the id could be overwritten. - evaluator.compiled_cache[key] = result, obj, parent, module - return result - return wrapper +_SPECIAL_OBJECTS = { + 'FUNCTION_CLASS': type(load_module), + 'MODULE_CLASS': type(os), + 'GENERATOR_OBJECT': _a_generator(1.0), + 'BUILTINS': _builtins, +} -@compiled_objects_cache -def create(evaluator, obj, parent=builtin, module=None): +def get_special_object(evaluator, identifier): + obj = _SPECIAL_OBJECTS[identifier] + return create(evaluator, obj, parent=create(evaluator, _builtins)) + + +def compiled_objects_cache(attribute_name): + def decorator(func): + """ + This decorator caches just the ids, oopposed to caching the object itself. + Caching the id has the advantage that an object doesn't need to be + hashable. + """ + def wrapper(evaluator, obj, parent=None, module=None): + cache = getattr(evaluator, attribute_name) + # Do a very cheap form of caching here. + key = id(obj), id(parent) + try: + return cache[key][0] + except KeyError: + # TODO this whole decorator looks way too ugly and this if + # doesn't make it better. Find a more generic solution. + if parent or module: + result = func(evaluator, obj, parent, module) + else: + result = func(evaluator, obj) + # Need to cache all of them, otherwise the id could be overwritten. + cache[key] = result, obj, parent, module + return result + return wrapper + + return decorator + + +@compiled_objects_cache('compiled_cache') +def create(evaluator, obj, parent=None, module=None): """ A very weird interface class to this module. The more options provided the more acurate loading compiled objects is. """ + if inspect.ismodule(obj): + if parent is not None: + # Modules don't have parents, be careful with caching: recurse. + return create(evaluator, obj) + else: + if parent is None and obj != _builtins: + return create(evaluator, obj, create(evaluator, _builtins)) - if not inspect.ismodule(obj): faked = fake.get_faked(module and module.obj, obj) if faked is not None: faked.parent = parent return faked - try: - if parent == builtin and obj.__module__ in ('builtins', '__builtin__'): - return builtin.get_by_name(obj.__name__) - except AttributeError: - pass - - return CompiledObject(obj, parent) + return CompiledObject(evaluator, obj, parent) diff --git a/jedi/evaluate/compiled/fake.py b/jedi/evaluate/compiled/fake.py index 32c28b66..54785acf 100644 --- a/jedi/evaluate/compiled/fake.py +++ b/jedi/evaluate/compiled/fake.py @@ -6,16 +6,44 @@ mixing in Python code, the autocompletion should work much better for builtins. import os import inspect +import types -from jedi._compatibility import is_py3, builtins, unicode +from jedi._compatibility import is_py3, builtins, unicode, is_py34 from jedi.cache import memoize_function -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.parser import tree as pt from jedi.evaluate.helpers import FakeName modules = {} +MethodDescriptorType = type(str.replace) +# These are not considered classes and access is granted even though they have +# a __class__ attribute. +NOT_CLASS_TYPES = ( + types.BuiltinFunctionType, + types.CodeType, + types.FrameType, + types.FunctionType, + types.GeneratorType, + types.GetSetDescriptorType, + types.LambdaType, + types.MemberDescriptorType, + types.MethodType, + types.ModuleType, + types.TracebackType, + MethodDescriptorType +) + +if is_py3: + NOT_CLASS_TYPES += ( + types.MappingProxyType, + types.SimpleNamespace + ) + if is_py34: + NOT_CLASS_TYPES += (types.DynamicClassAttribute,) + + def _load_faked_module(module): module_name = module.__name__ if module_name == '__builtin__' and not is_py3: @@ -31,8 +59,8 @@ def _load_faked_module(module): except IOError: modules[module_name] = None return - grammar = load_grammar('grammar3.4') - module = Parser(grammar, unicode(source), module_name).module + grammar = load_grammar(version='3.4') + module = ParserWithRecovery(grammar, unicode(source), module_name).module modules[module_name] = module if module_name == 'builtins' and not is_py3: @@ -65,7 +93,15 @@ def get_module(obj): # Unfortunately in some cases like `int` there's no __module__ return builtins else: - return __import__(imp_plz) + if imp_plz is None: + # Happens for example in `(_ for _ in []).send.__module__`. + return builtins + else: + try: + return __import__(imp_plz) + except ImportError: + # __module__ can be something arbitrary that doesn't exist. + return builtins def _faked(module, obj, name): @@ -75,7 +111,7 @@ def _faked(module, obj, name): faked_mod = _load_faked_module(module) if faked_mod is None: - return + return None # Having the module as a `parser.representation.module`, we need to scan # for methods. @@ -84,23 +120,32 @@ def _faked(module, obj, name): return search_scope(faked_mod, obj.__name__) elif not inspect.isclass(obj): # object is a method or descriptor - cls = search_scope(faked_mod, obj.__objclass__.__name__) - if cls is None: - return - return search_scope(cls, obj.__name__) + try: + objclass = obj.__objclass__ + except AttributeError: + return None + else: + cls = search_scope(faked_mod, objclass.__name__) + if cls is None: + return None + return search_scope(cls, obj.__name__) else: if obj == module: return search_scope(faked_mod, name) else: - cls = search_scope(faked_mod, obj.__name__) + try: + cls_name = obj.__name__ + except AttributeError: + return None + cls = search_scope(faked_mod, cls_name) if cls is None: - return + return None return search_scope(cls, name) @memoize_function def get_faked(module, obj, name=None): - obj = obj.__class__ if is_class_instance(obj) else obj + obj = type(obj) if is_class_instance(obj) else obj result = _faked(module, obj, name) if result is None or isinstance(result, pt.Class): # We're not interested in classes. What we want is functions. @@ -119,7 +164,9 @@ def get_faked(module, obj, name=None): def is_class_instance(obj): """Like inspect.* methods.""" - return not (inspect.isclass(obj) or inspect.ismodule(obj) - or inspect.isbuiltin(obj) or inspect.ismethod(obj) - or inspect.ismethoddescriptor(obj) or inspect.iscode(obj) - or inspect.isgenerator(obj)) + try: + cls = obj.__class__ + except AttributeError: + return False + else: + return cls != type and not issubclass(cls, NOT_CLASS_TYPES) diff --git a/jedi/evaluate/compiled/fake/builtins.pym b/jedi/evaluate/compiled/fake/builtins.pym index 1283de00..1d5314bd 100644 --- a/jedi/evaluate/compiled/fake/builtins.pym +++ b/jedi/evaluate/compiled/fake/builtins.pym @@ -124,7 +124,7 @@ class list(): return self.__iterable[y] def pop(self): - return self.__iterable[-1] + return self.__iterable[int()] class tuple(): @@ -202,11 +202,29 @@ class dict(): except KeyError: return d + def values(self): + return self.__elements.values() + def setdefault(self, k, d): # TODO maybe also return the content return d +class enumerate(): + def __init__(self, sequence, start=0): + self.__sequence = sequence + + def __iter__(self): + for i in self.__sequence: + yield 1, i + + def __next__(self): + return next(self.__iter__()) + + def next(self): + return next(self.__iter__()) + + class reversed(): def __init__(self, sequence): self.__sequence = sequence diff --git a/jedi/evaluate/compiled/mixed.py b/jedi/evaluate/compiled/mixed.py new file mode 100644 index 00000000..96bd33bc --- /dev/null +++ b/jedi/evaluate/compiled/mixed.py @@ -0,0 +1,158 @@ +""" +Used only for REPL Completion. +""" + +import inspect +import os + +from jedi import common +from jedi.parser.fast import FastParser +from jedi.evaluate import compiled +from jedi.cache import underscore_memoization + + +class MixedObject(object): + """ + A ``MixedObject`` is used in two ways: + + 1. It uses the default logic of ``parser.tree`` objects, + 2. except for getattr calls. The names dicts are generated in a fashion + like ``CompiledObject``. + + This combined logic makes it possible to provide more powerful REPL + completion. It allows side effects that are not noticable with the default + parser structure to still be completeable. + + The biggest difference from CompiledObject to MixedObject is that we are + generally dealing with Python code and not with C code. This will generate + fewer special cases, because we in Python you don't have the same freedoms + to modify the runtime. + """ + def __init__(self, evaluator, obj, node_name): + self._evaluator = evaluator + self.obj = obj + self.node_name = node_name + self.definition = node_name.get_definition() + + @property + def names_dict(self): + return LazyMixedNamesDict(self._evaluator, self) + + def names_dicts(self, search_global): + # TODO is this needed? + assert search_global is False + return [self.names_dict] + + def api_type(self): + mappings = { + 'expr_stmt': 'statement', + 'classdef': 'class', + 'funcdef': 'function', + 'file_input': 'module', + } + return mappings[self.definition.type] + + def __repr__(self): + return '<%s: %s>' % (type(self).__name__, repr(self.obj)) + + def __getattr__(self, name): + return getattr(self.definition, name) + + +class MixedName(compiled.CompiledName): + """ + The ``CompiledName._compiled_object`` is our MixedObject. + """ + @property + @underscore_memoization + def parent(self): + return create(self._evaluator, getattr(self._compiled_obj.obj, self.name)) + + @parent.setter + def parent(self, value): + pass # Just ignore this, Name tries to overwrite the parent attribute. + + @property + def start_pos(self): + if isinstance(self.parent, MixedObject): + return self.parent.node_name.start_pos + + # This means a start_pos that doesn't exist (compiled objects). + return (0, 0) + + +class LazyMixedNamesDict(compiled.LazyNamesDict): + name_class = MixedName + + +def parse(grammar, path): + with open(path) as f: + source = f.read() + source = common.source_to_unicode(source) + return FastParser(grammar, source, path) + + +def _load_module(evaluator, path, python_object): + module = parse(evaluator.grammar, path).module + python_module = inspect.getmodule(python_object) + + evaluator.modules[python_module.__name__] = module + return module + + +def find_syntax_node_name(evaluator, python_object): + try: + path = inspect.getsourcefile(python_object) + except TypeError: + # The type might not be known (e.g. class_with_dict.__weakref__) + return None + if path is None or not os.path.exists(path): + # The path might not exist or be e.g. . + return None + + module = _load_module(evaluator, path, python_object) + + if inspect.ismodule(python_object): + # We don't need to check names for modules, because there's not really + # a way to write a module in a module in Python (and also __name__ can + # be something like ``email.utils``). + return module + + try: + names = module.used_names[python_object.__name__] + except NameError: + return None + + names = [n for n in names if n.is_definition()] + + try: + code = python_object.__code__ + # By using the line number of a code object we make the lookup in a + # file pretty easy. There's still a possibility of people defining + # stuff like ``a = 3; foo(a); a = 4`` on the same line, but if people + # do so we just don't care. + line_nr = code.co_firstlineno + except AttributeError: + pass + else: + line_names = [name for name in names if name.start_pos[0] == line_nr] + # There's a chance that the object is not available anymore, because + # the code has changed in the background. + if line_names: + return line_names[-1] + + # It's really hard to actually get the right definition, here as a last + # resort we just return the last one. This chance might lead to odd + # completions at some points but will lead to mostly correct type + # inference, because people tend to define a public name in a module only + # once. + return names[-1] + + +@compiled.compiled_objects_cache('mixed_cache') +def create(evaluator, obj): + name = find_syntax_node_name(evaluator, obj) + if name is None: + return compiled.create(evaluator, obj) + else: + return MixedObject(evaluator, obj, name) diff --git a/jedi/evaluate/docstrings.py b/jedi/evaluate/docstrings.py index 84137de5..3561f619 100644 --- a/jedi/evaluate/docstrings.py +++ b/jedi/evaluate/docstrings.py @@ -20,7 +20,7 @@ from itertools import chain from textwrap import dedent from jedi.evaluate.cache import memoize_default -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.common import indent_block from jedi.evaluate.iterable import Array, FakeSequence, AlreadyEvaluated @@ -130,7 +130,7 @@ def _evaluate_for_statement_string(evaluator, string, module): # Take the default grammar here, if we load the Python 2.7 grammar here, it # will be impossible to use `...` (Ellipsis) as a token. Docstring types # don't need to conform with the current grammar. - p = Parser(load_grammar(), code % indent_block(string)) + p = ParserWithRecovery(load_grammar(), code % indent_block(string)) try: pseudo_cls = p.module.subscopes[0] # First pick suite, then simple_stmt (-2 for DEDENT) and then the node, @@ -164,8 +164,8 @@ def _execute_array_values(evaluator, array): """ if isinstance(array, Array): values = [] - for typ in array.values(): - objects = _execute_array_values(evaluator, typ) + for types in array.py__iter__(): + objects = set(chain.from_iterable(_execute_array_values(evaluator, typ) for typ in types)) values.append(AlreadyEvaluated(objects)) return [FakeSequence(evaluator, values, array.type)] else: @@ -176,11 +176,11 @@ def _execute_array_values(evaluator, array): def follow_param(evaluator, param): func = param.parent_function - return [p - for param_str in _search_param_in_docstr(func.raw_doc, - str(param.name)) + return set( + [p for param_str in _search_param_in_docstr(func.raw_doc, + str(param.name)) for p in _evaluate_for_statement_string(evaluator, param_str, - param.get_parent_until())] + param.get_parent_until())]) @memoize_default(None, evaluator_is_first_arg=True) diff --git a/jedi/evaluate/dynamic.py b/jedi/evaluate/dynamic.py index 04ed909a..d0570b59 100644 --- a/jedi/evaluate/dynamic.py +++ b/jedi/evaluate/dynamic.py @@ -26,6 +26,9 @@ from jedi.evaluate.cache import memoize_default from jedi.evaluate import imports +MAX_PARAM_SEARCHES = 20 + + class ParamListener(object): """ This listener is used to get the params for a function. @@ -52,17 +55,21 @@ def search_params(evaluator, param): is. """ if not settings.dynamic_params: - return [] + return set() - func = param.get_parent_until(tree.Function) - debug.dbg('Dynamic param search for %s in %s.', param, str(func.name)) - # Compare the param names. - names = [n for n in search_function_call(evaluator, func) - if n.value == param.name.value] - # Evaluate the ExecutedParams to types. - result = list(chain.from_iterable(n.parent.eval(evaluator) for n in names)) - debug.dbg('Dynamic param result %s', result) - return result + evaluator.dynamic_params_depth += 1 + try: + func = param.get_parent_until(tree.Function) + debug.dbg('Dynamic param search for %s in %s.', param, str(func.name), color='MAGENTA') + # Compare the param names. + names = [n for n in search_function_call(evaluator, func) + if n.value == param.name.value] + # Evaluate the ExecutedParams to types. + result = set(chain.from_iterable(n.parent.eval(evaluator) for n in names)) + debug.dbg('Dynamic param result %s', result, color='MAGENTA') + return result + finally: + evaluator.dynamic_params_depth -= 1 @memoize_default([], evaluator_is_first_arg=True) @@ -72,52 +79,29 @@ def search_function_call(evaluator, func): """ from jedi.evaluate import representation as er - def get_params_for_module(module): - """ - Returns the values of a param, or an empty array. - """ - @memoize_default([], evaluator_is_first_arg=True) - def get_posibilities(evaluator, module, func_name): + def get_possible_nodes(module, func_name): try: names = module.used_names[func_name] except KeyError: - return [] + return for name in names: - parent = name.parent - if tree.is_node(parent, 'trailer'): - parent = parent.parent + bracket = name.get_next_leaf() + trailer = bracket.parent + if trailer.type == 'trailer' and bracket == '(': + yield name, trailer - trailer = None - if tree.is_node(parent, 'power'): - for t in parent.children[1:]: - if t == '**': - break - if t.start_pos > name.start_pos and t.children[0] == '(': - trailer = t - break - if trailer is not None: - types = evaluator.goto_definition(name) - - # We have to remove decorators, because they are not the - # "original" functions, this way we can easily compare. - # At the same time we also have to remove InstanceElements. - undec = [] - for escope in types: - if escope.isinstance(er.Function, er.Instance) \ - and escope.decorates is not None: - undec.append(escope.decorates) - elif isinstance(escope, er.InstanceElement): - undec.append(escope.var) - else: - undec.append(escope) - - if evaluator.wrap(compare) in undec: - # Only if we have the correct function we execute - # it, otherwise just ignore it. - evaluator.eval_trailer(types, trailer) - return listener.param_possibilities - return get_posibilities(evaluator, module, func_name) + def undecorate(typ): + # We have to remove decorators, because they are not the + # "original" functions, this way we can easily compare. + # At the same time we also have to remove InstanceElements. + if typ.isinstance(er.Function, er.Instance) \ + and typ.decorates is not None: + return typ.decorates + elif isinstance(typ, er.InstanceElement): + return typ.var + else: + return typ current_module = func.get_parent_until() func_name = unicode(func.name) @@ -134,13 +118,32 @@ def search_function_call(evaluator, func): try: result = [] - # This is like backtracking: Get the first possible result. + i = 0 for mod in imports.get_modules_containing_name(evaluator, [current_module], func_name): - result = get_params_for_module(mod) + for name, trailer in get_possible_nodes(mod, func_name): + i += 1 + + # This is a simple way to stop Jedi's dynamic param recursion + # from going wild: The deeper Jedi's in the recursin, the less + # code should be evaluated. + if i * evaluator.dynamic_params_depth > MAX_PARAM_SEARCHES: + return listener.param_possibilities + + for typ in evaluator.goto_definitions(name): + undecorated = undecorate(typ) + if evaluator.wrap(compare) == undecorated: + # Only if we have the correct function we execute + # it, otherwise just ignore it. + evaluator.eval_trailer([typ], trailer) + + result = listener.param_possibilities + + # If there are results after processing a module, we're probably + # good to process. if result: - break + return result finally: # cleanup: remove the listener; important: should not stick. func.listeners.remove(listener) - return result + return set() diff --git a/jedi/evaluate/finder.py b/jedi/evaluate/finder.py index c112f971..2095959e 100644 --- a/jedi/evaluate/finder.py +++ b/jedi/evaluate/finder.py @@ -3,6 +3,9 @@ Searching for names with given scope and name. This is very central in Jedi and Python. The name resolution is quite complicated with descripter, ``__getattribute__``, ``__getattr__``, ``global``, etc. +If you want to understand name resolution, please read the first few chapters +in http://blog.ionelmc.ro/2015/02/09/understanding-python-metaclasses/. + Flow checks +++++++++++ @@ -13,15 +16,17 @@ check for -> a is a string). There's big potential in these checks. """ from itertools import chain -from jedi._compatibility import unicode, u +from jedi._compatibility import unicode from jedi.parser import tree from jedi import debug from jedi import common +from jedi.common import unite from jedi import settings from jedi.evaluate import representation as er from jedi.evaluate import dynamic from jedi.evaluate import compiled from jedi.evaluate import docstrings +from jedi.evaluate import pep0484 from jedi.evaluate import iterable from jedi.evaluate import imports from jedi.evaluate import analysis @@ -53,12 +58,15 @@ def filter_definition_names(names, origin, position=None): Filter names that are actual definitions in a scope. Names that are just used will be ignored. """ + if not names: + return [] + # Just calculate the scope from the first stmt = names[0].get_definition() scope = stmt.get_parent_scope() - if not (isinstance(scope, er.FunctionExecution) - and isinstance(scope.base, er.LambdaWrapper)): + if not (isinstance(scope, er.FunctionExecution) and + isinstance(scope.base, er.LambdaWrapper)): names = filter_after_position(names, position) names = [name for name in names if name.is_definition()] @@ -79,25 +87,34 @@ class NameFinder(object): self.scope = evaluator.wrap(scope) self.name_str = name_str self.position = position + self._found_predefined_if_name = None @debug.increase_indent - def find(self, scopes, search_global=False): + def find(self, scopes, attribute_lookup): + """ + :params bool attribute_lookup: Tell to logic if we're accessing the + attribute or the contents of e.g. a function. + """ # TODO rename scopes to names_dicts + names = self.filter_name(scopes) - types = self._names_to_types(names, search_global) + if self._found_predefined_if_name is not None: + return self._found_predefined_if_name + + types = self._names_to_types(names, attribute_lookup) if not names and not types \ - and not (isinstance(self.name_str, tree.Name) - and isinstance(self.name_str.parent.parent, tree.Param)): + and not (isinstance(self.name_str, tree.Name) and + isinstance(self.name_str.parent.parent, tree.Param)): if not isinstance(self.name_str, (str, unicode)): # TODO Remove? - if search_global: + if attribute_lookup: + analysis.add_attribute_error(self._evaluator, + self.scope, self.name_str) + else: message = ("NameError: name '%s' is not defined." % self.name_str) analysis.add(self._evaluator, 'name-error', self.name_str, message) - else: - analysis.add_attribute_error(self._evaluator, - self.scope, self.name_str) debug.dbg('finder._names_to_types: %s -> %s', names, types) return types @@ -143,6 +160,13 @@ class NameFinder(object): last_names.append(name) continue + if isinstance(stmt, er.ModuleWrapper): + # In case of REPL completion, we can infer modules names that + # don't really have a definition (because they are really just + # namespaces). In this case we can just add it. + last_names.append(name) + continue + if isinstance(name, compiled.CompiledName) \ or isinstance(name, er.InstanceName) and isinstance(name._origin_name, compiled.CompiledName): last_names.append(name) @@ -150,8 +174,35 @@ class NameFinder(object): if isinstance(self.name_str, tree.Name): origin_scope = self.name_str.get_parent_until(tree.Scope, reverse=True) + scope = self.name_str + check = None + while True: + scope = scope.parent + if scope.type in ("if_stmt", "for_stmt", "comp_for"): + try: + name_dict = self._evaluator.predefined_if_name_dict_dict[scope] + types = set(name_dict[str(self.name_str)]) + except KeyError: + continue + else: + if self.name_str.start_pos < scope.children[1].end_pos: + # It doesn't make any sense to check if + # statements in the if statement itself, just + # deliver types. + self._found_predefined_if_name = types + else: + check = flow_analysis.break_check(self._evaluator, self.scope, + origin_scope) + if check is flow_analysis.UNREACHABLE: + self._found_predefined_if_name = set() + else: + self._found_predefined_if_name = types + break + if isinstance(scope, tree.IsScope) or scope is None: + break else: origin_scope = None + if isinstance(stmt.parent, compiled.CompiledObject): # TODO seriously? this is stupid. continue @@ -159,6 +210,7 @@ class NameFinder(object): stmt, origin_scope) if check is not flow_analysis.UNREACHABLE: last_names.append(name) + if check is flow_analysis.REACHABLE: break @@ -179,7 +231,7 @@ class NameFinder(object): break debug.dbg('finder.filter_name "%s" in (%s): %s@%s', self.name_str, - self.scope, u(names), self.position) + self.scope, names, self.position) return list(self._clean_names(names)) def _clean_names(self, names): @@ -190,29 +242,37 @@ class NameFinder(object): """ for n in names: definition = n.parent - if isinstance(definition, (tree.Function, tree.Class, tree.Module)): + if isinstance(definition, (compiled.CompiledObject, + iterable.BuiltinMethod)): + # TODO this if should really be removed by changing the type of + # those classes. + yield n + elif definition.type in ('funcdef', 'classdef', 'file_input'): yield self._evaluator.wrap(definition).name else: yield n def _check_getattr(self, inst): """Checks for both __getattr__ and __getattribute__ methods""" - result = [] + result = set() # str is important, because it shouldn't be `Name`! name = compiled.create(self._evaluator, str(self.name_str)) with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattr__', name) if not result: - # this is a little bit special. `__getattribute__` is executed - # before anything else. But: I know no use case, where this - # could be practical and the jedi would return wrong types. If - # you ever have something, let me know! + # This is a little bit special. `__getattribute__` is in Python + # executed before `__getattr__`. But: I know no use case, where + # this could be practical and where jedi would return wrong types. + # If you ever find something, let me know! + # We are inversing this, because a hand-crafted `__getattribute__` + # could still call another hand-crafted `__getattr__`, but not the + # other way around. with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattribute__', name) return result - def _names_to_types(self, names, search_global): - types = [] + def _names_to_types(self, names, attribute_lookup): + types = set() # Add isinstance and other if/assert knowledge. if isinstance(self.name_str, tree.Name): @@ -231,13 +291,13 @@ class NameFinder(object): for name in names: new_types = _name_to_types(self._evaluator, name, self.scope) - if isinstance(self.scope, (er.Class, er.Instance)) and not search_global: - types += self._resolve_descriptors(name, new_types) + if isinstance(self.scope, (er.Class, er.Instance)) and attribute_lookup: + types |= set(self._resolve_descriptors(name, new_types)) else: - types += new_types + types |= set(new_types) if not names and isinstance(self.scope, er.Instance): # handling __getattr__ / __getattribute__ - types = self._check_getattr(self.scope) + return self._check_getattr(self.scope) return types @@ -249,56 +309,67 @@ class NameFinder(object): if not isinstance(name_scope, (er.Instance, tree.Class)): return types - result = [] + result = set() for r in types: try: desc_return = r.get_descriptor_returns except AttributeError: - result.append(r) + result.add(r) else: - result += desc_return(self.scope) + result |= desc_return(self.scope) return result -@memoize_default([], evaluator_is_first_arg=True) +def _get_global_stmt_scopes(evaluator, global_stmt, name): + global_stmt_scope = global_stmt.get_parent_scope() + module = global_stmt_scope.get_parent_until() + for used_name in module.used_names[str(name)]: + if used_name.parent.type == 'global_stmt': + yield evaluator.wrap(used_name.get_parent_scope()) + + +@memoize_default(set(), evaluator_is_first_arg=True) def _name_to_types(evaluator, name, scope): types = [] typ = name.get_definition() if typ.isinstance(tree.ForStmt): - for_types = evaluator.eval_element(typ.children[3]) - for_types = iterable.get_iterator_types(for_types) - types += check_tuple_assignments(for_types, name) - elif typ.isinstance(tree.CompFor): - for_types = evaluator.eval_element(typ.children[3]) - for_types = iterable.get_iterator_types(for_types) - types += check_tuple_assignments(for_types, name) + types = pep0484.find_type_from_comment_hint_for(evaluator, typ, name) + if types: + return types + if typ.isinstance(tree.WithStmt): + types = pep0484.find_type_from_comment_hint_with(evaluator, typ, name) + if types: + return types + if typ.isinstance(tree.ForStmt, tree.CompFor): + container_types = evaluator.eval_element(typ.children[3]) + for_types = iterable.py__iter__types(evaluator, container_types, typ.children[3]) + types = check_tuple_assignments(evaluator, for_types, name) elif isinstance(typ, tree.Param): - types += _eval_param(evaluator, typ, scope) + types = _eval_param(evaluator, typ, scope) elif typ.isinstance(tree.ExprStmt): - types += _remove_statements(evaluator, typ, name) + types = _remove_statements(evaluator, typ, name) elif typ.isinstance(tree.WithStmt): - types += evaluator.eval_element(typ.node_from_name(name)) + types = evaluator.eval_element(typ.node_from_name(name)) elif isinstance(typ, tree.Import): - types += imports.ImportWrapper(evaluator, name).follow() - elif isinstance(typ, tree.GlobalStmt): - # TODO theoretically we shouldn't be using search_global here, it - # doesn't make sense, because it's a local search (for that name)! - # However, globals are not that important and resolving them doesn't - # guarantee correctness in any way, because we don't check for when - # something is executed. - types += evaluator.find_types(typ.get_parent_scope(), str(name), - search_global=True) + types = imports.ImportWrapper(evaluator, name).follow() + elif typ.type == 'global_stmt': + for s in _get_global_stmt_scopes(evaluator, typ, name): + finder = NameFinder(evaluator, s, str(name)) + names_dicts = finder.scopes(search_global=True) + # For global_stmt lookups, we only need the first possible scope, + # which means the function itself. + names_dicts = [next(names_dicts)] + types += finder.find(names_dicts, attribute_lookup=False) elif isinstance(typ, tree.TryStmt): # TODO an exception can also be a tuple. Check for those. # TODO check for types that are not classes and add it to # the static analysis report. - exceptions = evaluator.eval_element(name.prev_sibling().prev_sibling()) - types = list(chain.from_iterable( - evaluator.execute(t) for t in exceptions)) + exceptions = evaluator.eval_element(name.get_previous_sibling().get_previous_sibling()) + types = set(chain.from_iterable(evaluator.execute(t) for t in exceptions)) else: if typ.isinstance(er.Function): typ = typ.get_decorated_func() - types.append(typ) + types = set([typ]) return types @@ -309,7 +380,7 @@ def _remove_statements(evaluator, stmt, name): Due to lazy evaluation, statements like a = func; b = a; b() have to be evaluated. """ - types = [] + types = set() # Remove the statement docstr stuff for now, that has to be # implemented with the evaluator class. #if stmt.docstr: @@ -320,18 +391,22 @@ def _remove_statements(evaluator, stmt, name): check_instance = stmt.instance stmt = stmt.var - types += evaluator.eval_statement(stmt, seek_name=name) + pep0484types = \ + pep0484.find_type_from_comment_hint_assign(evaluator, stmt, name) + if pep0484types: + return pep0484types + types |= evaluator.eval_statement(stmt, seek_name=name) if check_instance is not None: # class renames - types = [er.get_instance_el(evaluator, check_instance, a, True) - if isinstance(a, (er.Function, tree.Function)) - else a for a in types] + types = set([er.get_instance_el(evaluator, check_instance, a, True) + if isinstance(a, (er.Function, tree.Function)) + else a for a in types]) return types def _eval_param(evaluator, param, scope): - res_new = [] + res_new = set() func = param.get_parent_scope() cls = func.parent.get_parent_until((tree.Class, tree.Function)) @@ -342,11 +417,11 @@ def _eval_param(evaluator, param, scope): # This is where we add self - if it has never been # instantiated. if isinstance(scope, er.InstanceElement): - res_new.append(scope.instance) + res_new.add(scope.instance) else: inst = er.Instance(evaluator, evaluator.wrap(cls), Arguments(evaluator, ()), is_generated=True) - res_new.append(inst) + res_new.add(inst) return res_new # Instances are typically faked, if the instance is not called from @@ -355,23 +430,24 @@ def _eval_param(evaluator, param, scope): and func.instance.is_generated and str(func.name) == '__init__': param = func.var.params[param.position_nr] - # Add docstring knowledge. + # Add pep0484 and docstring knowledge. + pep0484_hints = pep0484.follow_param(evaluator, param) doc_params = docstrings.follow_param(evaluator, param) - if doc_params: - return doc_params + if pep0484_hints or doc_params: + return list(set(pep0484_hints) | set(doc_params)) if isinstance(param, ExecutedParam): - return res_new + param.eval(evaluator) + return res_new | param.eval(evaluator) else: # Param owns no information itself. - res_new += dynamic.search_params(evaluator, param) + res_new |= dynamic.search_params(evaluator, param) if not res_new: if param.stars: t = 'tuple' if param.stars == 1 else 'dict' - typ = evaluator.find_types(compiled.builtin, t)[0] + typ = list(evaluator.find_types(evaluator.BUILTINS, t))[0] res_new = evaluator.execute(typ) if param.default: - res_new += evaluator.eval_element(param.default) + res_new |= evaluator.eval_element(param.default) return res_new @@ -387,7 +463,7 @@ def check_flow_information(evaluator, flow, search_name, pos): if not settings.dynamic_flow_information: return None - result = [] + result = set() if flow.is_scope(): # Check for asserts. try: @@ -403,14 +479,16 @@ def check_flow_information(evaluator, flow, search_name, pos): break if isinstance(flow, (tree.IfStmt, tree.WhileStmt)): - element = flow.children[1] - result = _check_isinstance_type(evaluator, element, search_name) + potential_ifs = [c for c in flow.children[1::4] if c != ':'] + for if_test in reversed(potential_ifs): + if search_name.start_pos > if_test.end_pos: + return _check_isinstance_type(evaluator, if_test, search_name) return result def _check_isinstance_type(evaluator, element, search_name): try: - assert element.type == 'power' + assert element.type in ('power', 'atom_expr') # this might be removed if we analyze and, etc assert len(element.children) == 2 first, trailer = element.children @@ -428,15 +506,18 @@ def _check_isinstance_type(evaluator, element, search_name): # Do a simple get_code comparison. They should just have the same code, # and everything will be all right. classes = lst[1][1][0] - call = helpers.call_of_name(search_name) - assert name.get_code() == call.get_code() + call = helpers.call_of_leaf(search_name) + assert name.get_code(normalized=True) == call.get_code(normalized=True) except AssertionError: - return [] + return set() - result = [] - for typ in evaluator.eval_element(classes): - for typ in (typ.values() if isinstance(typ, iterable.Array) else [typ]): - result += evaluator.execute(typ) + result = set() + for cls_or_tup in evaluator.eval_element(classes): + if isinstance(cls_or_tup, iterable.Array) and cls_or_tup.type == 'tuple': + for typ in unite(cls_or_tup.py__iter__()): + result |= evaluator.execute(typ) + else: + result |= evaluator.execute(cls_or_tup) return result @@ -449,8 +530,8 @@ def global_names_dict_generator(evaluator, scope, position): the current scope is function: >>> from jedi._compatibility import u, no_unicode_pprint - >>> from jedi.parser import Parser, load_grammar - >>> parser = Parser(load_grammar(), u(''' + >>> from jedi.parser import ParserWithRecovery, load_grammar + >>> parser = ParserWithRecovery(load_grammar(), u(''' ... x = ['a', 'b', 'c'] ... def func(): ... y = None @@ -499,6 +580,12 @@ def global_names_dict_generator(evaluator, scope, position): for names_dict in scope.names_dicts(True): yield names_dict, position + if hasattr(scope, 'resets_positions'): + # TODO This is so ugly, seriously. However there's + # currently no good way of influencing + # global_names_dict_generator when it comes to certain + # objects. + position = None if scope.type == 'funcdef': # The position should be reset if the current scope is a function. in_func = True @@ -506,28 +593,26 @@ def global_names_dict_generator(evaluator, scope, position): scope = evaluator.wrap(scope.get_parent_scope()) # Add builtins to the global scope. - for names_dict in compiled.builtin.names_dicts(True): + for names_dict in evaluator.BUILTINS.names_dicts(True): yield names_dict, None -def check_tuple_assignments(types, name): +def check_tuple_assignments(evaluator, types, name): """ Checks if tuples are assigned. """ - for index in name.assignment_indexes(): - new_types = [] - for r in types: + for index, node in name.assignment_indexes(): + iterated = iterable.py__iter__(evaluator, types, node) + for _ in range(index + 1): try: - func = r.get_exact_index_types - except AttributeError: - debug.warning("Invalid tuple lookup #%s of result %s in %s", - index, types, name) - else: - try: - new_types += func(index) - except IndexError: - pass - types = new_types + types = next(iterated) + except StopIteration: + # We could do this with the default param in next. But this + # would allow this loop to run for a very long time if the + # index number is high. Therefore break if the loop is + # finished. + types = set() + break return types diff --git a/jedi/evaluate/flow_analysis.py b/jedi/evaluate/flow_analysis.py index cd3df554..e188264b 100644 --- a/jedi/evaluate/flow_analysis.py +++ b/jedi/evaluate/flow_analysis.py @@ -45,7 +45,8 @@ def break_check(evaluator, base_scope, stmt, origin_scope=None): if element_scope == origin_scope: return REACHABLE origin_scope = origin_scope.parent - return _break_check(evaluator, stmt, base_scope, element_scope) + x = _break_check(evaluator, stmt, base_scope, element_scope) + return x def _break_check(evaluator, stmt, base_scope, element_scope): @@ -62,7 +63,8 @@ def _break_check(evaluator, stmt, base_scope, element_scope): reachable = reachable.invert() else: node = element_scope.node_in_which_check_node(stmt) - reachable = _check_if(evaluator, node) + if node is not None: + reachable = _check_if(evaluator, node) elif isinstance(element_scope, (tree.TryStmt, tree.WhileStmt)): return UNSURE @@ -70,9 +72,14 @@ def _break_check(evaluator, stmt, base_scope, element_scope): if reachable in (UNREACHABLE, UNSURE): return reachable + if element_scope.type == 'file_input': + # The definition is in another module and therefore just return what we + # have generated. + return reachable if base_scope != element_scope and base_scope != element_scope.parent: return reachable & _break_check(evaluator, stmt, base_scope, element_scope.parent) - return reachable + else: + return reachable def _check_if(evaluator, node): diff --git a/jedi/evaluate/helpers.py b/jedi/evaluate/helpers.py index 4802bee0..b13e8dcb 100644 --- a/jedi/evaluate/helpers.py +++ b/jedi/evaluate/helpers.py @@ -26,7 +26,8 @@ def deep_ast_copy(obj, parent=None, new_elements=None): new_children = [] for child in obj.children: typ = child.type - if typ in ('whitespace', 'operator', 'keyword', 'number', 'string'): + if typ in ('whitespace', 'operator', 'keyword', 'number', 'string', + 'indent', 'dedent', 'error_leaf'): # At the moment we're not actually copying those primitive # elements, because there's really no need to. The parents are # obviously wrong, but that's not an issue. @@ -55,23 +56,18 @@ def deep_ast_copy(obj, parent=None, new_elements=None): new_names_dict[string] = [new_elements[n] for n in names] return new_obj - if obj.type == 'name': + if isinstance(obj, tree.BaseNode): + new_obj = copy_node(obj) + else: # Special case of a Name object. new_elements[obj] = new_obj = copy.copy(obj) - if parent is not None: - new_obj.parent = parent - elif isinstance(obj, tree.BaseNode): - new_obj = copy_node(obj) - if parent is not None: - for child in new_obj.children: - if isinstance(child, (tree.Name, tree.BaseNode)): - child.parent = parent - else: # String literals and so on. - new_obj = obj # Good enough, don't need to copy anything. + + if parent is not None: + new_obj.parent = parent return new_obj -def call_of_name(name, cut_own_trailer=False): +def call_of_leaf(leaf, cut_own_trailer=False): """ Creates a "call" node that consist of all ``trailer`` and ``power`` objects. E.g. if you call it with ``append``:: @@ -81,28 +77,53 @@ def call_of_name(name, cut_own_trailer=False): You would get a node with the content ``list([]).append`` back. This generates a copy of the original ast node. + + If you're using the leaf, e.g. the bracket `)` it will return ``list([])``. + + # TODO remove cut_own_trailer option, since its always used with it. Just + # ignore it, It's not what we want anyway. Or document it better? """ - par = name - if tree.is_node(par.parent, 'trailer'): - par = par.parent + trailer = leaf.parent + # The leaf may not be the last or first child, because there exist three + # different trailers: `( x )`, `[ x ]` and `.x`. In the first two examples + # we should not match anything more than x. + if trailer.type != 'trailer' or leaf not in (trailer.children[0], trailer.children[-1]): + if trailer.type == 'atom': + return trailer + return leaf - power = par.parent - if tree.is_node(power, 'power') and power.children[0] != name \ - and not (power.children[-2] == '**' and - name.start_pos > power.children[-1].start_pos): - par = power - # Now the name must be part of a trailer - index = par.children.index(name.parent) - if index != len(par.children) - 1 or cut_own_trailer: - # Now we have to cut the other trailers away. - par = deep_ast_copy(par) - if not cut_own_trailer: - # Normally we would remove just the stuff after the index, but - # if the option is set remove the index as well. (for goto) - index = index + 1 - par.children[index:] = [] + power = trailer.parent + index = power.children.index(trailer) + power = deep_ast_copy(power) + if cut_own_trailer: + cut = index + else: + cut = index + 1 + power.children[cut:] = [] - return par + if power.type == 'error_node': + start = index + while True: + start -= 1 + if power.children[start].type != 'trailer': + break + transformed = tree.Node('power', power.children[start:]) + transformed.parent = power.parent + return transformed + + return power + + +def get_names_of_node(node): + try: + children = node.children + except AttributeError: + if node.type == 'name': + return [node] + else: + return [] + else: + return list(chain.from_iterable(get_names_of_node(c) for c in children)) def get_module_names(module, all_scopes): diff --git a/jedi/evaluate/imports.py b/jedi/evaluate/imports.py index 3c226866..9f5842bd 100644 --- a/jedi/evaluate/imports.py +++ b/jedi/evaluate/imports.py @@ -20,9 +20,9 @@ from itertools import chain from jedi._compatibility import find_module, unicode from jedi import common from jedi import debug -from jedi import cache from jedi.parser import fast from jedi.parser import tree +from jedi.parser.utils import save_parser, load_parser, parser_cache from jedi.evaluate import sys_path from jedi.evaluate import helpers from jedi import settings @@ -70,7 +70,7 @@ class ImportWrapper(tree.Base): def follow(self, is_goto=False): if self._evaluator.recursion_detector.push_stmt(self._import): # check recursion - return [] + return set() try: module = self._evaluator.wrap(self._import.get_parent_until()) @@ -97,7 +97,7 @@ class ImportWrapper(tree.Base): # scopes = [NestedImportModule(module, self._import)] if from_import_name is not None: - types = list(chain.from_iterable( + types = set(chain.from_iterable( self._evaluator.find_types(t, unicode(from_import_name), is_goto=is_goto) for t in types)) @@ -109,11 +109,11 @@ class ImportWrapper(tree.Base): types = importer.follow() # goto only accepts `Name` if is_goto: - types = [s.name for s in types] + types = set(s.name for s in types) else: # goto only accepts `Name` if is_goto: - types = [s.name for s in types] + types = set(s.name for s in types) debug.dbg('after import: %s', types) finally: @@ -201,23 +201,24 @@ class Importer(object): base = [] if level > len(base): path = module.py__file__() - import_path = list(import_path) - for i in range(level): - path = os.path.dirname(path) - dir_name = os.path.basename(path) - # This is not the proper way to do relative imports. However, since - # Jedi cannot be sure about the entry point, we just calculate an - # absolute path here. - if dir_name: - import_path.insert(0, dir_name) - else: - _add_error(self._evaluator, import_path[-1]) - import_path = [] - # TODO add import error. - debug.warning('Attempted relative import beyond top-level package.') + if path is not None: + import_path = list(import_path) + for i in range(level): + path = os.path.dirname(path) + dir_name = os.path.basename(path) + # This is not the proper way to do relative imports. However, since + # Jedi cannot be sure about the entry point, we just calculate an + # absolute path here. + if dir_name: + import_path.insert(0, dir_name) + else: + _add_error(self._evaluator, import_path[-1]) + import_path = [] + # TODO add import error. + debug.warning('Attempted relative import beyond top-level package.') else: # Here we basically rewrite the level to 0. - import_path = tuple(base) + import_path + import_path = tuple(base) + tuple(import_path) self.import_path = import_path @property @@ -248,7 +249,7 @@ class Importer(object): @memoize_default(NO_DEFAULT) def follow(self): if not self.import_path: - return [] + return set() return self._do_import(self.import_path, self.sys_path_with_modifications()) def _do_import(self, import_path, sys_path): @@ -271,7 +272,7 @@ class Importer(object): module_name = '.'.join(import_parts) try: - return [self._evaluator.modules[module_name]] + return set([self._evaluator.modules[module_name]]) except KeyError: pass @@ -280,11 +281,11 @@ class Importer(object): # the module cache. bases = self._do_import(import_path[:-1], sys_path) if not bases: - return [] + return set() # We can take the first element, because only the os special # case yields multiple modules, which is not important for # further imports. - base = bases[0] + base = list(bases)[0] # This is a huge exception, we follow a nested import # ``os.path``, because it's a very important one in Python @@ -301,7 +302,7 @@ class Importer(object): except AttributeError: # The module is not a package. _add_error(self._evaluator, import_path[-1]) - return [] + return set() else: debug.dbg('search_module %s in paths %s', module_name, paths) for path in paths: @@ -315,7 +316,7 @@ class Importer(object): module_path = None if module_path is None: _add_error(self._evaluator, import_path[-1]) - return [] + return set() else: try: debug.dbg('search_module %s in %s', import_parts[-1], self.file_path) @@ -330,7 +331,7 @@ class Importer(object): except ImportError: # The module is not a package. _add_error(self._evaluator, import_path[-1]) - return [] + return set() source = None if is_pkg: @@ -346,11 +347,20 @@ class Importer(object): else: module = _load_module(self._evaluator, module_path, source, sys_path) + if module is None: + # The file might raise an ImportError e.g. and therefore not be + # importable. + return set() + self._evaluator.modules[module_name] = module - return [module] + return set([module]) def _generate_name(self, name): - return helpers.FakeName(name, parent=self.module) + # Create a pseudo import to be able to follow them. + name = helpers.FakeName(name) + imp = helpers.FakeImport(name, parent=self.module) + name.parent = imp + return name def _get_module_names(self, search_path=None): """ @@ -435,7 +445,7 @@ def _load_module(evaluator, path=None, source=None, sys_path=None): def load(source): dotted_path = path and compiled.dotted_from_fs_path(path, sys_path) if path is not None and path.endswith('.py') \ - and not dotted_path in settings.auto_import_modules: + and dotted_path not in settings.auto_import_modules: if source is None: with open(path, 'rb') as f: source = f.read() @@ -443,13 +453,13 @@ def _load_module(evaluator, path=None, source=None, sys_path=None): return compiled.load_module(evaluator, path) p = path p = fast.FastParser(evaluator.grammar, common.source_to_unicode(source), p) - cache.save_parser(path, p) + save_parser(path, p) return p.module if sys_path is None: sys_path = evaluator.sys_path - cached = cache.load_parser(path) + cached = load_parser(path) module = load(source) if cached is None else cached.module module = evaluator.wrap(module) return module @@ -470,7 +480,7 @@ def get_modules_containing_name(evaluator, mods, name): """ def check_python_file(path): try: - return cache.parser_cache[path].parser.module + return parser_cache[path].parser.module except KeyError: try: return check_fs(path) @@ -497,7 +507,9 @@ def get_modules_containing_name(evaluator, mods, name): paths = set(settings.additional_dynamic_modules) for p in mod_paths: if p is not None: - d = os.path.dirname(p) + # We need abspath, because the seetings paths might not already + # have been converted to absolute paths. + d = os.path.dirname(os.path.abspath(p)) for entry in os.listdir(d): if entry not in mod_paths: if entry.endswith('.py'): diff --git a/jedi/evaluate/iterable.py b/jedi/evaluate/iterable.py index d14a4c5f..3bbd7141 100644 --- a/jedi/evaluate/iterable.py +++ b/jedi/evaluate/iterable.py @@ -20,74 +20,117 @@ It is important to note that: 1. Array modfications work only in the current module. 2. Jedi only checks Array additions; ``list.pop``, etc are ignored. """ -from itertools import chain - -from jedi import common +from jedi.common import unite, safe_property from jedi import debug from jedi import settings -from jedi._compatibility import use_metaclass, is_py3, unicode +from jedi._compatibility import use_metaclass, unicode, zip_longest from jedi.parser import tree from jedi.evaluate import compiled from jedi.evaluate import helpers from jedi.evaluate.cache import CachedMetaClass, memoize_default from jedi.evaluate import analysis - - -def unite(iterable): - """Turns a two dimensional array into a one dimensional.""" - return list(chain.from_iterable(iterable)) +from jedi.evaluate import pep0484 class IterableWrapper(tree.Base): def is_class(self): return False + @memoize_default() + def _get_names_dict(self, names_dict): + builtin_methods = {} + for cls in reversed(type(self).mro()): + try: + builtin_methods.update(cls.builtin_methods) + except AttributeError: + pass + if not builtin_methods: + return names_dict + + dct = {} + for names in names_dict.values(): + for name in names: + name_str = name.value + try: + method = builtin_methods[name_str, self.type] + except KeyError: + dct[name_str] = [name] + else: + parent = BuiltinMethod(self, method, name.parent) + dct[name_str] = [helpers.FakeName(name_str, parent, is_definition=True)] + return dct + + +class BuiltinMethod(IterableWrapper): + """``Generator.__next__`` ``dict.values`` methods and so on.""" + def __init__(self, builtin, method, builtin_func): + self._builtin = builtin + self._method = method + self._builtin_func = builtin_func + + def py__call__(self, params): + return self._method(self._builtin) + + def __getattr__(self, name): + return getattr(self._builtin_func, name) + + +def has_builtin_methods(cls): + cls.builtin_methods = {} + for func in cls.__dict__.values(): + try: + cls.builtin_methods.update(func.registered_builtin_methods) + except AttributeError: + pass + return cls + + +def register_builtin_method(method_name, type=None): + def wrapper(func): + dct = func.__dict__.setdefault('registered_builtin_methods', {}) + dct[method_name, type] = func + return func + return wrapper + + +@has_builtin_methods class GeneratorMixin(object): + type = None + + @register_builtin_method('send') + @register_builtin_method('next') + @register_builtin_method('__next__') + def py__next__(self): + # TODO add TypeError if params are given. + return unite(self.py__iter__()) + @memoize_default() def names_dicts(self, search_global=False): # is always False - dct = {} - executes_generator = '__next__', 'send', 'next' - for names in compiled.generator_obj.names_dict.values(): - for name in names: - if name.value in executes_generator: - parent = GeneratorMethod(self, name.parent) - dct[name.value] = [helpers.FakeName(name.name, parent, is_definition=True)] - else: - dct[name.value] = [name] - yield dct - - def get_index_types(self, evaluator, index_array): - #debug.warning('Tried to get array access on a generator: %s', self) - analysis.add(self._evaluator, 'type-error-generator', index_array) - return [] - - def get_exact_index_types(self, index): - """ - Exact lookups are used for tuple lookups, which are perfectly fine if - used with generators. - """ - return [self.iter_content()[index]] + gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT') + yield self._get_names_dict(gen_obj.names_dict) def py__bool__(self): return True + def py__class__(self): + gen_obj = compiled.get_special_object(self._evaluator, 'GENERATOR_OBJECT') + return gen_obj.py__class__() + class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)): """Handling of `yield` functions.""" + def __init__(self, evaluator, func, var_args): super(Generator, self).__init__() self._evaluator = evaluator self.func = func self.var_args = var_args - def iter_content(self): - """ returns the content of __iter__ """ - # Directly execute it, because with a normal call to py__call__ a - # Generator will be returned. + def py__iter__(self): from jedi.evaluate.representation import FunctionExecution f = FunctionExecution(self._evaluator, self.func, self.var_args) - return f.get_return_types(check_yields=True) + return f.get_yield_types() def __getattr__(self, name): if name not in ['start_pos', 'end_pos', 'parent', 'get_imports', @@ -101,90 +144,172 @@ class Generator(use_metaclass(CachedMetaClass, IterableWrapper, GeneratorMixin)) return "<%s of %s>" % (type(self).__name__, self.func) -class GeneratorMethod(IterableWrapper): - """``__next__`` and ``send`` methods.""" - def __init__(self, generator, builtin_func): - self._builtin_func = builtin_func - self._generator = generator - - def py__call__(self, evaluator, params): - # TODO add TypeError if params are given. - return self._generator.iter_content() - - def __getattr__(self, name): - return getattr(self._builtin_func, name) - - class Comprehension(IterableWrapper): @staticmethod def from_atom(evaluator, atom): - mapping = { - '(': GeneratorComprehension, - '[': ListComprehension - } - return mapping[atom.children[0]](evaluator, atom) + bracket = atom.children[0] + if bracket == '{': + if atom.children[1].children[1] == ':': + cls = DictComprehension + else: + cls = SetComprehension + elif bracket == '(': + cls = GeneratorComprehension + elif bracket == '[': + cls = ListComprehension + return cls(evaluator, atom) def __init__(self, evaluator, atom): self._evaluator = evaluator self._atom = atom + def _get_comprehension(self): + # The atom contains a testlist_comp + return self._atom.children[1] + + def _get_comp_for(self): + # The atom contains a testlist_comp + return self._get_comprehension().children[1] + @memoize_default() - def eval_node(self): + def _eval_node(self, index=0): """ The first part `x + 1` of the list comprehension: [x + 1 for x in foo] """ - comprehension = self._atom.children[1] + comp_for = self._get_comp_for() # For nested comprehensions we need to search the last one. - last = comprehension.children[-1] - last_comp = comprehension.children[1] - while True: - if isinstance(last, tree.CompFor): - last_comp = last - elif not tree.is_node(last, 'comp_if'): - break - last = last.children[-1] + last_comp = list(comp_for.get_comp_fors())[-1] + return helpers.deep_ast_copy(self._get_comprehension().children[index], parent=last_comp) - return helpers.deep_ast_copy(comprehension.children[0], parent=last_comp) + @memoize_default() + def _iterate(self): + def nested(comp_fors): + comp_for = comp_fors[0] + input_node = comp_for.children[3] + input_types = evaluator.eval_element(input_node) - def get_exact_index_types(self, index): - return [self._evaluator.eval_element(self.eval_node())[index]] + iterated = py__iter__(evaluator, input_types, input_node) + exprlist = comp_for.children[1] + for types in iterated: + evaluator.predefined_if_name_dict_dict[comp_for] = \ + unpack_tuple_to_dict(evaluator, types, exprlist) + try: + for result in nested(comp_fors[1:]): + yield result + except IndexError: + iterated = evaluator.eval_element(self._eval_node()) + if self.type == 'dict': + yield iterated, evaluator.eval_element(self._eval_node(2)) + else: + yield iterated + finally: + del evaluator.predefined_if_name_dict_dict[comp_for] + + evaluator = self._evaluator + comp_fors = list(self._get_comp_for().get_comp_fors()) + for result in nested(comp_fors): + yield result + + def py__iter__(self): + return self._iterate() def __repr__(self): - return "" % (type(self).__name__, self._atom) + return "<%s of %s>" % (type(self).__name__, self._atom) +@has_builtin_methods class ArrayMixin(object): @memoize_default() def names_dicts(self, search_global=False): # Always False. # `array.type` is a string with the type, e.g. 'list'. - scope = self._evaluator.find_types(compiled.builtin, self.type)[0] + scope = compiled.builtin_from_name(self._evaluator, self.type) # builtins only have one class -> [0] - scope = self._evaluator.execute(scope, (AlreadyEvaluated((self,)),))[0] - return scope.names_dicts(search_global) + scopes = self._evaluator.execute_evaluated(scope, self) + names_dicts = list(scopes)[0].names_dicts(search_global) + #yield names_dicts[0] + yield self._get_names_dict(names_dicts[1]) def py__bool__(self): return None # We don't know the length, because of appends. + def py__class__(self): + return compiled.builtin_from_name(self._evaluator, self.type) + + @safe_property + def parent(self): + return self._evaluator.BUILTINS + + @property + def name(self): + return FakeSequence(self._evaluator, [], self.type).name + + @memoize_default() + def dict_values(self): + return unite(self._evaluator.eval_element(v) for k, v in self._items()) + + @register_builtin_method('values', type='dict') + def _imitate_values(self): + items = self.dict_values() + return create_evaluated_sequence_set(self._evaluator, items, type='list') + #return set([FakeSequence(self._evaluator, [AlreadyEvaluated(items)], 'tuple')]) + + @register_builtin_method('items', type='dict') + def _imitate_items(self): + items = [set([FakeSequence(self._evaluator, (k, v), 'tuple')]) + for k, v in self._items()] + + return create_evaluated_sequence_set(self._evaluator, *items, type='list') + class ListComprehension(Comprehension, ArrayMixin): type = 'list' - def get_index_types(self, evaluator, index): - return self.iter_content() + def py__getitem__(self, index): + all_types = list(self.py__iter__()) + return all_types[index] + + +class SetComprehension(Comprehension, ArrayMixin): + type = 'set' + + +@has_builtin_methods +class DictComprehension(Comprehension, ArrayMixin): + type = 'dict' + + def _get_comp_for(self): + return self._get_comprehension().children[3] + + def py__iter__(self): + for keys, values in self._iterate(): + yield keys + + def py__getitem__(self, index): + for keys, values in self._iterate(): + for k in keys: + if isinstance(k, compiled.CompiledObject): + if k.obj == index: + return values + return self.dict_values() + + def dict_values(self): + return unite(values for keys, values in self._iterate()) + + @register_builtin_method('items', type='dict') + def _imitate_items(self): + items = set(FakeSequence(self._evaluator, + (AlreadyEvaluated(keys), AlreadyEvaluated(values)), 'tuple') + for keys, values in self._iterate()) + + return create_evaluated_sequence_set(self._evaluator, items, type='list') - def iter_content(self): - return self._evaluator.eval_element(self.eval_node()) - @property - def name(self): - return FakeSequence(self._evaluator, [], 'list').name class GeneratorComprehension(Comprehension, GeneratorMixin): - def iter_content(self): - return self._evaluator.eval_element(self.eval_node()) + pass class Array(IterableWrapper, ArrayMixin): @@ -209,60 +334,21 @@ class Array(IterableWrapper, ArrayMixin): def name(self): return helpers.FakeName(self.type, parent=self) - @memoize_default() - def get_index_types(self, evaluator, index=()): - """ - Get the types of a specific index or all, if not given. - - :param index: A subscriptlist node (or subnode). - """ - indexes = create_indexes_or_slices(evaluator, index) - lookup_done = False - types = [] - for index in indexes: - if isinstance(index, Slice): - types += [self] - lookup_done = True - elif isinstance(index, compiled.CompiledObject) \ - and isinstance(index.obj, (int, str, unicode)): - with common.ignored(KeyError, IndexError, TypeError): - types += self.get_exact_index_types(index.obj) - lookup_done = True - - return types if lookup_done else self.values() - - @memoize_default() - def values(self): - result = unite(self._evaluator.eval_element(v) for v in self._values()) - result += check_array_additions(self._evaluator, self) - return result - - def get_exact_index_types(self, mixed_index): - """ Here the index is an int/str. Raises IndexError/KeyError """ + def py__getitem__(self, index): + """Here the index is an int/str. Raises IndexError/KeyError.""" if self.type == 'dict': - for key, values in self._items(): - # Because we only want the key to be a string. - keys = self._evaluator.eval_element(key) - - for k in keys: + for key, value in self._items(): + for k in self._evaluator.eval_element(key): if isinstance(k, compiled.CompiledObject) \ - and mixed_index == k.obj: - for value in values: - return self._evaluator.eval_element(value) + and index == k.obj: + return self._evaluator.eval_element(value) raise KeyError('No key found in dictionary %s.' % self) # Can raise an IndexError - return self._evaluator.eval_element(self._items()[mixed_index]) - - def iter_content(self): - return self.values() - - @common.safe_property - def parent(self): - return compiled.builtin - - def get_parent_until(self): - return compiled.builtin + if isinstance(index, slice): + return set([self]) + else: + return self._evaluator.eval_element(self._items()[index]) def __getattr__(self, name): if name not in ['start_pos', 'get_only_subelement', 'parent', @@ -270,10 +356,33 @@ class Array(IterableWrapper, ArrayMixin): raise AttributeError('Strange access on %s: %s.' % (self, name)) return getattr(self.atom, name) + # @memoize_default() + def py__iter__(self): + """ + While values returns the possible values for any array field, this + function returns the value for a certain index. + """ + if self.type == 'dict': + # Get keys. + types = set() + for k, _ in self._items(): + types |= self._evaluator.eval_element(k) + # We don't know which dict index comes first, therefore always + # yield all the types. + for _ in types: + yield types + else: + for value in self._items(): + yield self._evaluator.eval_element(value) + + additions = check_array_additions(self._evaluator, self) + if additions: + yield additions + def _values(self): """Returns a list of a list of node.""" if self.type == 'dict': - return list(chain.from_iterable(v for k, v in self._items())) + return unite(v for k, v in self._items()) else: return self._items() @@ -292,18 +401,14 @@ class Array(IterableWrapper, ArrayMixin): op = next(iterator, None) if op is None or op == ',': kv.append(key) # A set. - elif op == ':': # A dict. - kv.append((key, [next(iterator)])) - next(iterator, None) # Possible comma. else: - raise NotImplementedError('dict/set comprehensions') + assert op == ':' # A dict. + kv.append((key, next(iterator))) + next(iterator, None) # Possible comma. return kv else: return [array_node] - def __iter__(self): - return iter(self._items()) - def __repr__(self): return "<%s of %s>" % (type(self).__name__, self.atom) @@ -326,20 +431,29 @@ class ImplicitTuple(_FakeArray): class FakeSequence(_FakeArray): def __init__(self, evaluator, sequence_values, type): + """ + type should be one of "tuple", "list" + """ super(FakeSequence, self).__init__(evaluator, sequence_values, type) self._sequence_values = sequence_values def _items(self): return self._sequence_values - def get_exact_index_types(self, index): - value = self._sequence_values[index] - return self._evaluator.eval_element(value) + +def create_evaluated_sequence_set(evaluator, *types_order, **kwargs): + """ + ``sequence_type`` is a named argument, that doesn't work in Python2. For backwards + compatibility reasons, we're now using kwargs. + """ + sequence_type = kwargs.get('sequence_type') + sets = tuple(AlreadyEvaluated(types) for types in types_order) + return set([FakeSequence(evaluator, sets, sequence_type)]) class AlreadyEvaluated(frozenset): """A simple container to add already evaluated objects to an array.""" - def get_code(self): + def get_code(self, normalized=False): # For debugging purposes. return str(self) @@ -353,12 +467,16 @@ class FakeDict(_FakeArray): super(FakeDict, self).__init__(evaluator, dct, 'dict') self._dct = dct - def get_exact_index_types(self, index): - return list(chain.from_iterable(self._evaluator.eval_element(v) - for v in self._dct[index])) + def py__iter__(self): + yield set(compiled.create(self._evaluator, key) for key in self._dct) + + def py__getitem__(self, index): + return unite(self._evaluator.eval_element(v) for v in self._dct[index]) def _items(self): - return self._dct.items() + for key, values in self._dct.items(): + # TODO this is not proper. The values could be multiple values?! + yield key, values[0] class MergedArray(_FakeArray): @@ -366,56 +484,138 @@ class MergedArray(_FakeArray): super(MergedArray, self).__init__(evaluator, arrays, arrays[-1].type) self._arrays = arrays - def get_exact_index_types(self, mixed_index): - raise IndexError - - def values(self): - return list(chain(*(a.values() for a in self._arrays))) - - def __iter__(self): + def py__iter__(self): for array in self._arrays: - for a in array: + for types in array.py__iter__(): + yield types + + def py__getitem__(self, index): + return unite(self.py__iter__()) + + def _items(self): + for array in self._arrays: + for a in array._items(): yield a def __len__(self): return sum(len(a) for a in self._arrays) -def get_iterator_types(inputs): - """Returns the types of any iterator (arrays, yields, __iter__, etc).""" - iterators = [] - # Take the first statement (for has always only - # one, remember `in`). And follow it. - for it in inputs: - if isinstance(it, (Generator, Array, ArrayInstance, Comprehension)): - iterators.append(it) - else: - if not hasattr(it, 'execute_subscope_by_name'): - debug.warning('iterator/for loop input wrong: %s', it) - continue +def unpack_tuple_to_dict(evaluator, types, exprlist): + """ + Unpacking tuple assignments in for statements and expr_stmts. + """ + if exprlist.type == 'name': + return {exprlist.value: types} + elif exprlist.type == 'atom' and exprlist.children[0] in '([': + return unpack_tuple_to_dict(evaluator, types, exprlist.children[1]) + elif exprlist.type in ('testlist', 'testlist_comp', 'exprlist', + 'testlist_star_expr'): + dct = {} + parts = iter(exprlist.children[::2]) + n = 0 + for iter_types in py__iter__(evaluator, types, exprlist): + n += 1 try: - iterators += it.execute_subscope_by_name('__iter__') - except KeyError: - debug.warning('iterators: No __iter__ method found.') + part = next(parts) + except StopIteration: + analysis.add(evaluator, 'value-error-too-many-values', part, + message="ValueError: too many values to unpack (expected %s)" % n) + else: + dct.update(unpack_tuple_to_dict(evaluator, iter_types, part)) + has_parts = next(parts, None) + if types and has_parts is not None: + analysis.add(evaluator, 'value-error-too-few-values', has_parts, + message="ValueError: need more than %s values to unpack" % n) + return dct + elif exprlist.type == 'power' or exprlist.type == 'atom_expr': + # Something like ``arr[x], var = ...``. + # This is something that is not yet supported, would also be difficult + # to write into a dict. + return {} + elif exprlist.type == 'star_expr': # `a, *b, c = x` type unpackings + # Currently we're not supporting them. + return {} + raise NotImplementedError - result = [] - from jedi.evaluate.representation import Instance - for it in iterators: - if isinstance(it, Array): - # Array is a little bit special, since this is an internal array, - # but there's also the list builtin, which is another thing. - result += it.values() - elif isinstance(it, Instance): - # __iter__ returned an instance. - name = '__next__' if is_py3 else 'next' - try: - result += it.execute_subscope_by_name(name) - except KeyError: - debug.warning('Instance has no __next__ function in %s.', it) + +def py__iter__(evaluator, types, node=None): + debug.dbg('py__iter__') + type_iters = [] + for typ in types: + try: + iter_method = typ.py__iter__ + except AttributeError: + if node is not None: + analysis.add(evaluator, 'type-error-not-iterable', node, + message="TypeError: '%s' object is not iterable" % typ) else: - # TODO this is not correct, __iter__ can return arbitrary input! - # Is a generator. - result += it.iter_content() + type_iters.append(iter_method()) + #for result in iter_method(): + #yield result + + for t in zip_longest(*type_iters, fillvalue=set()): + yield unite(t) + + +def py__iter__types(evaluator, types, node=None): + """ + Calls `py__iter__`, but ignores the ordering in the end and just returns + all types that it contains. + """ + return unite(py__iter__(evaluator, types, node)) + + +def py__getitem__(evaluator, types, trailer): + from jedi.evaluate.representation import Class + result = set() + + trailer_op, node, trailer_cl = trailer.children + assert trailer_op == "[" + assert trailer_cl == "]" + + # special case: PEP0484 typing module, see + # https://github.com/davidhalter/jedi/issues/663 + for typ in list(types): + if isinstance(typ, Class): + typing_module_types = \ + pep0484.get_types_for_typing_module(evaluator, typ, node) + if typing_module_types is not None: + types.remove(typ) + result |= typing_module_types + + if not types: + # all consumed by special cases + return result + + for index in create_index_types(evaluator, node): + if isinstance(index, (compiled.CompiledObject, Slice)): + index = index.obj + + if type(index) not in (float, int, str, unicode, slice): + # If the index is not clearly defined, we have to get all the + # possiblities. + for typ in list(types): + if isinstance(typ, Array) and typ.type == 'dict': + types.remove(typ) + result |= typ.dict_values() + return result | py__iter__types(evaluator, types) + + for typ in types: + # The actual getitem call. + try: + getitem = typ.py__getitem__ + except AttributeError: + analysis.add(evaluator, 'type-error-not-subscriptable', trailer_op, + message="TypeError: '%s' object is not subscriptable" % typ) + else: + try: + result |= getitem(index) + except IndexError: + result |= py__iter__types(evaluator, set([typ])) + except KeyError: + # Must be a dict. Lists don't raise KeyErrors. + result |= typ.dict_values() return result @@ -423,7 +623,7 @@ def check_array_additions(evaluator, array): """ Just a mapper function for the internal _check_array_additions """ if array.type not in ('list', 'set'): # TODO also check for dict updates - return [] + return set() is_list = array.type == 'list' try: @@ -432,11 +632,12 @@ def check_array_additions(evaluator, array): # If there's no get_parent_until, it's a FakeSequence or another Fake # type. Those fake types are used inside Jedi's engine. No values may # be added to those after their creation. - return [] + return set() return _check_array_additions(evaluator, array, current_module, is_list) -@memoize_default([], evaluator_is_first_arg=True) +@memoize_default(default=set(), evaluator_is_first_arg=True) +@debug.increase_indent def _check_array_additions(evaluator, compare_array, module, is_list): """ Checks if a `Array` has "add" (append, insert, extend) statements: @@ -444,21 +645,24 @@ def _check_array_additions(evaluator, compare_array, module, is_list): >>> a = [""] >>> a.append(1) """ + debug.dbg('Dynamic array search for %s' % compare_array, color='MAGENTA') if not settings.dynamic_array_additions or isinstance(module, compiled.CompiledObject): - return [] + debug.dbg('Dynamic array search aborted.', color='MAGENTA') + return set() def check_additions(arglist, add_name): params = list(param.Arguments(evaluator, arglist).unpack()) - result = [] + result = set() if add_name in ['insert']: params = params[1:] if add_name in ['append', 'add', 'insert']: for key, nodes in params: - result += unite(evaluator.eval_element(node) for node in nodes) + result |= unite(evaluator.eval_element(node) for node in nodes) elif add_name in ['extend', 'update']: for key, nodes in params: - iterators = unite(evaluator.eval_element(node) for node in nodes) - result += get_iterator_types(iterators) + for node in nodes: + types = evaluator.eval_element(node) + result |= py__iter__types(evaluator, types, node) return result from jedi.evaluate import representation as er, param @@ -469,7 +673,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list): node = element.atom else: # Is an Instance with an - # Arguments([AlreadyEvaluated([ArrayInstance])]) inside + # Arguments([AlreadyEvaluated([_ArrayInstance])]) inside # Yeah... I know... It's complicated ;-) node = list(element.var_args.argument_node[0])[0].var_args.trailer if isinstance(node, er.InstanceElement): @@ -482,7 +686,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list): search_names = ['append', 'extend', 'insert'] if is_list else ['add', 'update'] comp_arr_parent = get_execution_parent(compare_array) - added_types = [] + added_types = set() for add_name in search_names: try: possible_names = module.used_names[add_name] @@ -515,7 +719,7 @@ def _check_array_additions(evaluator, compare_array, module, is_list): or execution_trailer.children[0] != '(' \ or execution_trailer.children[1] == ')': continue - power = helpers.call_of_name(name, cut_own_trailer=True) + power = helpers.call_of_leaf(name, cut_own_trailer=True) # InstanceElements are special, because they don't get copied, # but have this wrapper around them. if isinstance(comp_arr_parent, er.InstanceElement): @@ -527,11 +731,12 @@ def _check_array_additions(evaluator, compare_array, module, is_list): continue if compare_array in evaluator.eval_element(power): # The arrays match. Now add the results - added_types += check_additions(execution_trailer.children[1], add_name) + added_types |= check_additions(execution_trailer.children[1], add_name) evaluator.recursion_detector.pop_stmt() # reset settings settings.dynamic_params_for_other_modules = temp_param_add + debug.dbg('Dynamic array result %s' % added_types, color='MAGENTA') return added_types @@ -540,12 +745,12 @@ def check_array_instances(evaluator, instance): if not settings.dynamic_array_additions: return instance.var_args - ai = ArrayInstance(evaluator, instance) + ai = _ArrayInstance(evaluator, instance) from jedi.evaluate import param return param.Arguments(evaluator, [AlreadyEvaluated([ai])]) -class ArrayInstance(IterableWrapper): +class _ArrayInstance(IterableWrapper): """ Used for the usage of set() and list(). This is definitely a hack, but a good one :-) @@ -561,21 +766,21 @@ class ArrayInstance(IterableWrapper): self.instance = instance self.var_args = instance.var_args - def iter_content(self): - """ - The index is here just ignored, because of all the appends, etc. - lists/sets are too complicated too handle that. - """ - items = [] - for key, nodes in self.var_args.unpack(): - for node in nodes: - for typ in self._evaluator.eval_element(node): - items += get_iterator_types([typ]) + def py__iter__(self): + try: + _, first_nodes = next(self.var_args.unpack()) + except StopIteration: + types = set() + else: + types = unite(self._evaluator.eval_element(node) for node in first_nodes) + for types in py__iter__(self._evaluator, types, first_nodes[0]): + yield types module = self.var_args.get_parent_until() is_list = str(self.instance.name) == 'list' - items += _check_array_additions(self._evaluator, self.instance, module, is_list) - return items + additions = _check_array_additions(self._evaluator, self.instance, module, is_list) + if additions: + yield additions class Slice(object): @@ -598,11 +803,11 @@ class Slice(object): result = self._evaluator.eval_element(element) if len(result) != 1: - # We want slices to be clear defined with just one type. - # Otherwise we will return an empty slice object. + # For simplicity, we want slices to be clear defined with just + # one type. Otherwise we will return an empty slice object. raise IndexError try: - return result[0].obj + return list(result)[0].obj except AttributeError: return None @@ -612,9 +817,15 @@ class Slice(object): return slice(None, None, None) -def create_indexes_or_slices(evaluator, index): - if tree.is_node(index, 'subscript'): # subscript is a slice operation. - start, stop, step = None, None, None +def create_index_types(evaluator, index): + """ + Handles slices in subscript nodes. + """ + if index == ':': + # Like array[:] + return set([Slice(evaluator, None, None, None)]) + elif tree.is_node(index, 'subscript'): # subscript is a slice operation. + # Like array[:3] result = [] for el in index.children: if el == ':': @@ -627,5 +838,7 @@ def create_indexes_or_slices(evaluator, index): result.append(el) result += [None] * (3 - len(result)) - return (Slice(evaluator, *result),) + return set([Slice(evaluator, *result)]) + + # No slices return evaluator.eval_element(index) diff --git a/jedi/evaluate/jedi_typing.py b/jedi/evaluate/jedi_typing.py new file mode 100644 index 00000000..f48a5673 --- /dev/null +++ b/jedi/evaluate/jedi_typing.py @@ -0,0 +1,100 @@ +""" +This module is not intended to be used in jedi, rather it will be fed to the +jedi-parser to replace classes in the typing module +""" + +try: + from collections import abc +except ImportError: + # python 2 + import collections as abc + + +def factory(typing_name, indextypes): + class Iterable(abc.Iterable): + def __iter__(self): + while True: + yield indextypes[0]() + + class Iterator(Iterable, abc.Iterator): + def next(self): + """ needed for python 2 """ + return self.__next__() + + def __next__(self): + return indextypes[0]() + + class Sequence(abc.Sequence): + def __getitem__(self, index): + return indextypes[0]() + + class MutableSequence(Sequence, abc.MutableSequence): + pass + + class List(MutableSequence, list): + pass + + class Tuple(Sequence, tuple): + def __getitem__(self, index): + if indextypes[1] == Ellipsis: + # https://www.python.org/dev/peps/pep-0484/#the-typing-module + # Tuple[int, ...] means a tuple of ints of indetermined length + return indextypes[0]() + else: + return indextypes[index]() + + class AbstractSet(Iterable, abc.Set): + pass + + class MutableSet(AbstractSet, abc.MutableSet): + pass + + class KeysView(Iterable, abc.KeysView): + pass + + class ValuesView(abc.ValuesView): + def __iter__(self): + while True: + yield indextypes[1]() + + class ItemsView(abc.ItemsView): + def __iter__(self): + while True: + yield (indextypes[0](), indextypes[1]()) + + class Mapping(Iterable, abc.Mapping): + def __getitem__(self, item): + return indextypes[1]() + + def keys(self): + return KeysView() + + def values(self): + return ValuesView() + + def items(self): + return ItemsView() + + class MutableMapping(Mapping, abc.MutableMapping): + pass + + class Dict(MutableMapping, dict): + pass + + dct = { + "Sequence": Sequence, + "MutableSequence": MutableSequence, + "List": List, + "Iterable": Iterable, + "Iterator": Iterator, + "AbstractSet": AbstractSet, + "MutableSet": MutableSet, + "Mapping": Mapping, + "MutableMapping": MutableMapping, + "Tuple": Tuple, + "KeysView": KeysView, + "ItemsView": ItemsView, + "ValuesView": ValuesView, + "Dict": Dict, + } + return dct[typing_name] diff --git a/jedi/evaluate/param.py b/jedi/evaluate/param.py index 8524bf95..997e0799 100644 --- a/jedi/evaluate/param.py +++ b/jedi/evaluate/param.py @@ -12,6 +12,23 @@ from jedi.evaluate.helpers import FakeName from jedi.cache import underscore_memoization +def try_iter_content(types, depth=0): + """Helper method for static analysis.""" + if depth > 10: + # It's possible that a loop has references on itself (especially with + # CompiledObject). Therefore don't loop infinitely. + return + + for typ in types: + try: + f = typ.py__iter__ + except AttributeError: + pass + else: + for iter_types in f(): + try_iter_content(iter_types, depth + 1) + + class Arguments(tree.Base): def __init__(self, evaluator, argument_node, trailer=None): """ @@ -30,7 +47,10 @@ class Arguments(tree.Base): for el in self.argument_node: yield 0, el else: - if not tree.is_node(self.argument_node, 'arglist'): + if not (tree.is_node(self.argument_node, 'arglist') or ( + # in python 3.5 **arg is an argument, not arglist + (tree.is_node(self.argument_node, 'argument') and + self.argument_node.children[0] in ('*', '**')))): yield 0, self.argument_node return @@ -40,6 +60,10 @@ class Arguments(tree.Base): continue elif child in ('*', '**'): yield len(child.value), next(iterator) + elif tree.is_node(child, 'argument') and \ + child.children[0] in ('*', '**'): + assert len(child.children) == 2 + yield len(child.children[0].value), child.children[1] else: yield 0, child @@ -49,7 +73,7 @@ class Arguments(tree.Base): element = self.argument_node[0] from jedi.evaluate.iterable import AlreadyEvaluated if isinstance(element, AlreadyEvaluated): - element = self._evaluator.eval_element(element)[0] + element = list(self._evaluator.eval_element(element))[0] except IndexError: return None else: @@ -131,8 +155,8 @@ class Arguments(tree.Base): debug.warning('TypeError: %s expected at least %s arguments, got %s', name, len(arguments), i) raise ValueError - values = list(chain.from_iterable(self._evaluator.eval_element(el) - for el in va_values)) + values = set(chain.from_iterable(self._evaluator.eval_element(el) + for el in va_values)) if not values and not optional: # For the stdlib we always want values. If we don't get them, # that's ok, maybe something is too hard to resolve, however, @@ -160,6 +184,16 @@ class Arguments(tree.Base): else: return None + def eval_all(self, func=None): + """ + Evaluates all arguments as a support for static analysis + (normally Jedi). + """ + for key, element_values in self.unpack(): + for element in element_values: + types = self._evaluator.eval_element(element) + try_iter_content(types) + class ExecutedParam(tree.Param): """Fake a param and give it values.""" @@ -169,9 +203,9 @@ class ExecutedParam(tree.Param): self._values = values def eval(self, evaluator): - types = [] + types = set() for v in self._values: - types += evaluator.eval_element(v) + types |= evaluator.eval_element(v) return types @property @@ -242,7 +276,7 @@ def get_params(evaluator, func, var_args): try: key_param = param_dict[unicode(key)] except KeyError: - non_matching_keys[key] += va_values + non_matching_keys[key] = va_values else: param_names.append(ExecutedParam(key_param, var_args, va_values).name) @@ -353,11 +387,12 @@ def get_params(evaluator, func, var_args): def _iterate_star_args(evaluator, array, input_node, func=None): from jedi.evaluate.representation import Instance if isinstance(array, iterable.Array): - for field_stmt in array: # yield from plz! - yield field_stmt + # TODO ._items is not the call we want here. Replace in the future. + for node in array._items(): + yield node elif isinstance(array, iterable.Generator): - for field_stmt in array.iter_content(): - yield iterable.AlreadyEvaluated([field_stmt]) + for types in array.py__iter__(): + yield iterable.AlreadyEvaluated(types) elif isinstance(array, Instance) and array.name.get_code() == 'tuple': debug.warning('Ignored a tuple *args input %s' % array) else: @@ -379,10 +414,10 @@ def _star_star_dict(evaluator, array, input_node, func): return array._dct elif isinstance(array, iterable.Array) and array.type == 'dict': # TODO bad call to non-public API - for key_node, values in array._items(): + for key_node, value in array._items(): for key in evaluator.eval_element(key_node): if precedence.is_string(key): - dct[key.obj] += values + dct[key.obj].append(value) else: if func is not None: diff --git a/jedi/evaluate/pep0484.py b/jedi/evaluate/pep0484.py new file mode 100644 index 00000000..2387fe64 --- /dev/null +++ b/jedi/evaluate/pep0484.py @@ -0,0 +1,195 @@ +""" +PEP 0484 ( https://www.python.org/dev/peps/pep-0484/ ) describes type hints +through function annotations. There is a strong suggestion in this document +that only the type of type hinting defined in PEP0484 should be allowed +as annotations in future python versions. + +The (initial / probably incomplete) implementation todo list for pep-0484: +v Function parameter annotations with builtin/custom type classes +v Function returntype annotations with builtin/custom type classes +v Function parameter annotations with strings (forward reference) +v Function return type annotations with strings (forward reference) +v Local variable type hints +v Assigned types: `Url = str\ndef get(url:Url) -> str:` +v Type hints in `with` statements +x Stub files support +x support `@no_type_check` and `@no_type_check_decorator` +x support for typing.cast() operator +x support for type hint comments for functions, `# type: (int, str) -> int`. + See comment from Guido https://github.com/davidhalter/jedi/issues/662 +""" + +import itertools + +import os +from jedi.parser import \ + Parser, load_grammar, ParseError, ParserWithRecovery, tree +from jedi.evaluate.cache import memoize_default +from jedi.common import unite +from jedi.evaluate import compiled +from jedi import debug +from jedi import _compatibility +import re + + +def _evaluate_for_annotation(evaluator, annotation, index=None): + """ + Evaluates a string-node, looking for an annotation + If index is not None, the annotation is expected to be a tuple + and we're interested in that index + """ + if annotation is not None: + definitions = evaluator.eval_element( + _fix_forward_reference(evaluator, annotation)) + if index is not None: + definitions = list(itertools.chain.from_iterable( + definition.py__getitem__(index) for definition in definitions + if definition.type == 'tuple' and + len(list(definition.py__iter__())) >= index)) + return list(itertools.chain.from_iterable( + evaluator.execute(d) for d in definitions)) + else: + return [] + + +def _fix_forward_reference(evaluator, node): + evaled_nodes = evaluator.eval_element(node) + if len(evaled_nodes) != 1: + debug.warning("Eval'ed typing index %s should lead to 1 object, " + " not %s" % (node, evaled_nodes)) + return node + evaled_node = list(evaled_nodes)[0] + if isinstance(evaled_node, compiled.CompiledObject) and \ + isinstance(evaled_node.obj, str): + try: + p = Parser(load_grammar(), _compatibility.unicode(evaled_node.obj), + start_symbol='eval_input') + newnode = p.get_parsed_node() + except ParseError: + debug.warning('Annotation not parsed: %s' % evaled_node.obj) + return node + else: + module = node.get_parent_until() + p.position_modifier.line = module.end_pos[0] + newnode.parent = module + return newnode + else: + return node + + +@memoize_default(None, evaluator_is_first_arg=True) +def follow_param(evaluator, param): + annotation = param.annotation() + return _evaluate_for_annotation(evaluator, annotation) + + +@memoize_default(None, evaluator_is_first_arg=True) +def find_return_types(evaluator, func): + annotation = func.py__annotations__().get("return", None) + return _evaluate_for_annotation(evaluator, annotation) + + +_typing_module = None + + +def _get_typing_replacement_module(): + """ + The idea is to return our jedi replacement for the PEP-0484 typing module + as discussed at https://github.com/davidhalter/jedi/issues/663 + """ + global _typing_module + if _typing_module is None: + typing_path = \ + os.path.abspath(os.path.join(__file__, "../jedi_typing.py")) + with open(typing_path) as f: + code = _compatibility.unicode(f.read()) + p = ParserWithRecovery(load_grammar(), code) + _typing_module = p.module + return _typing_module + + +def get_types_for_typing_module(evaluator, typ, node): + from jedi.evaluate.iterable import FakeSequence + if not typ.base.get_parent_until().name.value == "typing": + return None + # we assume that any class using [] in a module called + # "typing" with a name for which we have a replacement + # should be replaced by that class. This is not 100% + # airtight but I don't have a better idea to check that it's + # actually the PEP-0484 typing module and not some other + if tree.is_node(node, "subscriptlist"): + nodes = node.children[::2] # skip the commas + else: + nodes = [node] + del node + + nodes = [_fix_forward_reference(evaluator, node) for node in nodes] + + # hacked in Union and Optional, since it's hard to do nicely in parsed code + if typ.name.value == "Union": + return unite(evaluator.eval_element(node) for node in nodes) + if typ.name.value == "Optional": + return evaluator.eval_element(nodes[0]) + + typing = _get_typing_replacement_module() + factories = evaluator.find_types(typing, "factory") + assert len(factories) == 1 + factory = list(factories)[0] + assert factory + function_body_nodes = factory.children[4].children + valid_classnames = set(child.name.value + for child in function_body_nodes + if isinstance(child, tree.Class)) + if typ.name.value not in valid_classnames: + return None + compiled_classname = compiled.create(evaluator, typ.name.value) + + args = FakeSequence(evaluator, nodes, "tuple") + + result = evaluator.execute_evaluated(factory, compiled_classname, args) + return result + + +def find_type_from_comment_hint_for(evaluator, node, name): + return \ + _find_type_from_comment_hint(evaluator, node, node.children[1], name) + + +def find_type_from_comment_hint_with(evaluator, node, name): + assert len(node.children[1].children) == 3, \ + "Can only be here when children[1] is 'foo() as f'" + return _find_type_from_comment_hint( + evaluator, node, node.children[1].children[2], name) + + +def find_type_from_comment_hint_assign(evaluator, node, name): + return \ + _find_type_from_comment_hint(evaluator, node, node.children[0], name) + + +def _find_type_from_comment_hint(evaluator, node, varlist, name): + index = None + if varlist.type in ("testlist_star_expr", "exprlist"): + # something like "a, b = 1, 2" + index = 0 + for child in varlist.children: + if child == name: + break + if child.type == "operator": + continue + index += 1 + else: + return [] + + comment = node.get_following_comment_same_line() + if comment is None: + return [] + match = re.match(r"^#\s*type:\s*([^#]*)", comment) + if not match: + return [] + annotation = tree.String( + tree.zero_position_modifier, + repr(str(match.group(1).strip())), + node.start_pos) + annotation.parent = node.parent + return _evaluate_for_annotation(evaluator, annotation, index) diff --git a/jedi/evaluate/precedence.py b/jedi/evaluate/precedence.py index 7a2ee6d2..5225aa68 100644 --- a/jedi/evaluate/precedence.py +++ b/jedi/evaluate/precedence.py @@ -6,8 +6,7 @@ import operator from jedi._compatibility import unicode from jedi.parser import tree from jedi import debug -from jedi.evaluate.compiled import (CompiledObject, create, builtin, - keyword_from_value, true_obj, false_obj) +from jedi.evaluate.compiled import CompiledObject, create, builtin_from_name from jedi.evaluate import analysis # Maps Python syntax to the operator module. @@ -23,16 +22,19 @@ COMPARISON_OPERATORS = { } -def _literals_to_types(evaluator, result): +def literals_to_types(evaluator, result): # Changes literals ('a', 1, 1.0, etc) to its type instances (str(), # int(), float(), etc). - for i, r in enumerate(result): - if is_literal(r): + new_result = set() + for typ in result: + if is_literal(typ): # Literals are only valid as long as the operations are # correct. Otherwise add a value-free instance. - cls = builtin.get_by_name(r.name.get_code()) - result[i] = evaluator.execute(cls)[0] - return list(set(result)) + cls = builtin_from_name(evaluator, typ.name.value) + new_result |= evaluator.execute(cls) + else: + new_result.add(typ) + return new_result def calculate_children(evaluator, children): @@ -64,21 +66,21 @@ def calculate_children(evaluator, children): def calculate(evaluator, left_result, operator, right_result): - result = [] + result = set() if not left_result or not right_result: # illegal slices e.g. cause left/right_result to be None - result = (left_result or []) + (right_result or []) - result = _literals_to_types(evaluator, result) + result = (left_result or set()) | (right_result or set()) + result = literals_to_types(evaluator, result) else: # I don't think there's a reasonable chance that a string # operation is still correct, once we pass something like six # objects. if len(left_result) * len(right_result) > 6: - result = _literals_to_types(evaluator, left_result + right_result) + result = literals_to_types(evaluator, left_result | right_result) else: for left in left_result: for right in right_result: - result += _element_calculate(evaluator, left, operator, right) + result |= _element_calculate(evaluator, left, operator, right) return result @@ -94,7 +96,7 @@ def factor_calculate(evaluator, types, operator): value = typ.py__bool__() if value is None: # Uncertainty. return - yield keyword_from_value(not value) + yield create(evaluator, not value) else: yield typ @@ -130,21 +132,21 @@ def _element_calculate(evaluator, left, operator, right): if operator == '*': # for iterables, ignore * operations if isinstance(left, iterable.Array) or is_string(left): - return [left] + return set([left]) elif isinstance(right, iterable.Array) or is_string(right): - return [right] + return set([right]) elif operator == '+': if l_is_num and r_is_num or is_string(left) and is_string(right): - return [create(evaluator, left.obj + right.obj)] + return set([create(evaluator, left.obj + right.obj)]) elif _is_tuple(left) and _is_tuple(right) or _is_list(left) and _is_list(right): - return [iterable.MergedArray(evaluator, (left, right))] + return set([iterable.MergedArray(evaluator, (left, right))]) elif operator == '-': if l_is_num and r_is_num: - return [create(evaluator, left.obj - right.obj)] + return set([create(evaluator, left.obj - right.obj)]) elif operator == '%': # With strings and numbers the left type typically remains. Except for # `int() % float()`. - return [left] + return set([left]) elif operator in COMPARISON_OPERATORS: operation = COMPARISON_OPERATORS[operator] if isinstance(left, CompiledObject) and isinstance(right, CompiledObject): @@ -153,12 +155,14 @@ def _element_calculate(evaluator, left, operator, right): right = right.obj try: - return [keyword_from_value(operation(left, right))] + result = operation(left, right) except TypeError: # Could be True or False. - return [true_obj, false_obj] + return set([create(evaluator, True), create(evaluator, False)]) + else: + return set([create(evaluator, result)]) elif operator == 'in': - return [] + return set() def check(obj): """Checks if a Jedi object is either a float or an int.""" @@ -171,4 +175,4 @@ def _element_calculate(evaluator, left, operator, right): analysis.add(evaluator, 'type-error-operation', operator, message % (left, right)) - return [left, right] + return set([left, right]) diff --git a/jedi/evaluate/recursion.py b/jedi/evaluate/recursion.py index a4f5fbc3..a95ba3e0 100644 --- a/jedi/evaluate/recursion.py +++ b/jedi/evaluate/recursion.py @@ -9,7 +9,6 @@ count the function calls. """ from jedi import debug from jedi import settings -from jedi.evaluate import compiled from jedi.evaluate import iterable @@ -18,7 +17,7 @@ def recursion_decorator(func): rec_detect = evaluator.recursion_detector # print stmt, len(self.node_statements()) if rec_detect.push_stmt(stmt): - return [] + return set() else: result = func(evaluator, stmt, *args, **kwargs) rec_detect.pop_stmt() @@ -31,12 +30,13 @@ class RecursionDetector(object): A decorator to detect recursions in statements. In a recursion a statement at the same place, in the same module may not be executed two times. """ - def __init__(self): + def __init__(self, evaluator): self.top = None self.current = None + self._evaluator = evaluator def push_stmt(self, stmt): - self.current = _RecursionNode(stmt, self.current) + self.current = _RecursionNode(self._evaluator, stmt, self.current) check = self._check_recursion() if check: debug.warning('catched stmt recursion: %s against %s @%s', stmt, @@ -71,7 +71,8 @@ class RecursionDetector(object): class _RecursionNode(object): """ A node of the RecursionDecorator. """ - def __init__(self, stmt, parent): + def __init__(self, evaluator, stmt, parent): + self._evaluator = evaluator self.script = stmt.get_parent_until() self.position = stmt.start_pos self.parent = parent @@ -80,7 +81,7 @@ class _RecursionNode(object): # Don't check param instances, they are not causing recursions # The same's true for the builtins, because the builtins are really # simple. - self.is_ignored = self.script == compiled.builtin + self.is_ignored = self.script == self._evaluator.BUILTINS def __eq__(self, other): if not other: @@ -95,7 +96,7 @@ def execution_recursion_decorator(func): def run(execution, **kwargs): detector = execution._evaluator.execution_recursion_detector if detector.push_execution(execution): - result = [] + result = set() else: result = func(execution, **kwargs) detector.pop_execution() @@ -107,51 +108,51 @@ def execution_recursion_decorator(func): class ExecutionRecursionDetector(object): """ Catches recursions of executions. - It is designed like a Singelton. Only one instance should exist. """ - def __init__(self): + def __init__(self, evaluator): self.recursion_level = 0 self.parent_execution_funcs = [] self.execution_funcs = set() self.execution_count = 0 + self._evaluator = evaluator def __call__(self, execution): debug.dbg('Execution recursions: %s', execution, self.recursion_level, self.execution_count, len(self.execution_funcs)) if self.check_recursion(execution): - result = [] + result = set() else: result = self.func(execution) self.pop_execution() return result - def pop_execution(cls): - cls.parent_execution_funcs.pop() - cls.recursion_level -= 1 + def pop_execution(self): + self.parent_execution_funcs.pop() + self.recursion_level -= 1 - def push_execution(cls, execution): - in_par_execution_funcs = execution.base in cls.parent_execution_funcs - in_execution_funcs = execution.base in cls.execution_funcs - cls.recursion_level += 1 - cls.execution_count += 1 - cls.execution_funcs.add(execution.base) - cls.parent_execution_funcs.append(execution.base) + def push_execution(self, execution): + in_par_execution_funcs = execution.base in self.parent_execution_funcs + in_execution_funcs = execution.base in self.execution_funcs + self.recursion_level += 1 + self.execution_count += 1 + self.execution_funcs.add(execution.base) + self.parent_execution_funcs.append(execution.base) - if cls.execution_count > settings.max_executions: + if self.execution_count > settings.max_executions: return True if isinstance(execution.base, (iterable.Array, iterable.Generator)): return False module = execution.get_parent_until() - if module == compiled.builtin: + if module == self._evaluator.BUILTINS: return False if in_par_execution_funcs: - if cls.recursion_level > settings.max_function_recursion_level: + if self.recursion_level > settings.max_function_recursion_level: return True if in_execution_funcs and \ - len(cls.execution_funcs) > settings.max_until_execution_unique: + len(self.execution_funcs) > settings.max_until_execution_unique: return True - if cls.execution_count > settings.max_executions_without_builtins: + if self.execution_count > settings.max_executions_without_builtins: return True return False diff --git a/jedi/evaluate/representation.py b/jedi/evaluate/representation.py index 3cfcaa9b..a95e5451 100644 --- a/jedi/evaluate/representation.py +++ b/jedi/evaluate/representation.py @@ -17,13 +17,18 @@ and others. Here's a list: ====================================== ======================================== **Method** **Description** -------------------------------------- ---------------------------------------- -py__call__(evaluator, params: Array) On callable objects, returns types. +py__call__(params: Array) On callable objects, returns types. py__bool__() Returns True/False/None; None means that there's no certainty. -py__bases__(evaluator) Returns a list of base classes. -py__mro__(evaluator) Returns a list of classes (the mro). -py__getattribute__(evaluator, name) Returns a list of attribute values. The - name can be str or Name. +py__bases__() Returns a list of base classes. +py__mro__() Returns a list of classes (the mro). +py__iter__() Returns a generator of a set of types. +py__class__() Returns the class of an instance. +py__getitem__(index: int/str) Returns a a set of types of the index. + Can raise an IndexError/KeyError. +py__file__() Only on modules. +py__package__() Only on modules. For the import system. +py__path__() Only on modules. For the import system. ====================================== ======================================== __ @@ -34,16 +39,18 @@ import imp import re from itertools import chain -from jedi._compatibility import use_metaclass, unicode, Python3Method +from jedi._compatibility import use_metaclass, unicode, Python3Method, is_py3 from jedi.parser import tree from jedi import debug from jedi import common from jedi.cache import underscore_memoization, cache_star_import from jedi.evaluate.cache import memoize_default, CachedMetaClass, NO_DEFAULT from jedi.evaluate import compiled +from jedi.evaluate.compiled import mixed from jedi.evaluate import recursion from jedi.evaluate import iterable from jedi.evaluate import docstrings +from jedi.evaluate import pep0484 from jedi.evaluate import helpers from jedi.evaluate import param from jedi.evaluate import flow_analysis @@ -83,7 +90,7 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): self.is_generated = is_generated if base.name.get_code() in ['list', 'set'] \ - and compiled.builtin == base.get_parent_until(): + and evaluator.BUILTINS == base.get_parent_until(): # compare the module path with the builtin name. self.var_args = iterable.check_array_instances(evaluator, self) elif not is_generated: @@ -96,10 +103,13 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): else: evaluator.execute(method, self.var_args) + def is_class(self): + return False + @property def py__call__(self): - def actual(evaluator, params): - return evaluator.execute(method, params) + def actual(params): + return self._evaluator.execute(method, params) try: method = self.get_subscope_by_name('__call__') @@ -109,7 +119,7 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): return actual - def py__class__(self, evaluator): + def py__class__(self): return self.base def py__bool__(self): @@ -153,8 +163,8 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): sub = self._get_method_execution(sub) for name_list in sub.names_dict.values(): for name in name_list: - if name.value == self_name and name.prev_sibling() is None: - trailer = name.next_sibling() + if name.value == self_name and name.get_previous_sibling() is None: + trailer = name.get_next_sibling() if tree.is_node(trailer, 'trailer') \ and len(trailer.children) == 2 \ and trailer.children[0] == '.': @@ -176,17 +186,18 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): """ Throws a KeyError if there's no method. """ # Arguments in __get__ descriptors are obj, class. # `method` is the new parent of the array, don't know if that's good. - args = [obj, obj.base] if isinstance(obj, Instance) else [compiled.none_obj, obj] + none_obj = compiled.create(self._evaluator, None) + args = [obj, obj.base] if isinstance(obj, Instance) else [none_obj, obj] try: return self.execute_subscope_by_name('__get__', *args) except KeyError: - return [self] + return set([self]) @memoize_default() def names_dicts(self, search_global): yield self._self_names_dict() - for s in self.base.py__mro__(self._evaluator)[1:]: + for s in self.base.py__mro__()[1:]: if not isinstance(s, compiled.CompiledObject): # Compiled objects don't have `self.` names. for inst in self._evaluator.execute(s): @@ -195,21 +206,35 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): for names_dict in self.base.names_dicts(search_global=False, is_instance=True): yield LazyInstanceDict(self._evaluator, self, names_dict) - def get_index_types(self, evaluator, index_array): - indexes = iterable.create_indexes_or_slices(self._evaluator, index_array) - if any([isinstance(i, iterable.Slice) for i in indexes]): - # Slice support in Jedi is very marginal, at the moment, so just - # ignore them in case of __getitem__. - # TODO support slices in a more general way. - indexes = [] - + def py__getitem__(self, index): try: method = self.get_subscope_by_name('__getitem__') except KeyError: debug.warning('No __getitem__, cannot access the array.') - return [] + return set() else: - return self._evaluator.execute(method, [iterable.AlreadyEvaluated(indexes)]) + index_obj = compiled.create(self._evaluator, index) + return self._evaluator.execute_evaluated(method, index_obj) + + def py__iter__(self): + try: + method = self.get_subscope_by_name('__iter__') + except KeyError: + debug.warning('No __iter__ on %s.' % self) + return + else: + iters = self._evaluator.execute(method) + for generator in iters: + if isinstance(generator, Instance): + # `__next__` logic. + name = '__next__' if is_py3 else 'next' + try: + yield generator.execute_subscope_by_name(name) + except KeyError: + debug.warning('Instance has no __next__ function in %s.', generator) + else: + for typ in generator.py__iter__(): + yield typ @property @underscore_memoization @@ -228,8 +253,8 @@ class Instance(use_metaclass(CachedMetaClass, Executed)): dec = '' if self.decorates is not None: dec = " decorates " + repr(self.decorates) - return "" % (type(self).__name__, self.base, - self.var_args, dec) + return "<%s of %s(%s)%s>" % (type(self).__name__, self.base, + self.var_args, dec) class LazyInstanceDict(object): @@ -355,13 +380,13 @@ class InstanceElement(use_metaclass(CachedMetaClass, tree.Base)): """ return self.var.is_scope() - def py__call__(self, evaluator, params): + def py__call__(self, params): if isinstance(self.var, compiled.CompiledObject): # This check is a bit strange, but CompiledObject itself is a bit # more complicated than we would it actually like to be. - return self.var.py__call__(evaluator, params) + return self.var.py__call__(params) else: - return Function.py__call__(self, evaluator, params) + return Function.py__call__(self, params) def __repr__(self): return "<%s of %s>" % (type(self).__name__, self.var) @@ -398,7 +423,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): self.base = base @memoize_default(default=()) - def py__mro__(self, evaluator): + def py__mro__(self): def add(cls): if cls not in mro: mro.append(cls) @@ -406,7 +431,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): mro = [self] # TODO Do a proper mro resolution. Currently we are just listing # classes. However, it's a complicated algorithm. - for cls in self.py__bases__(self._evaluator): + for cls in self.py__bases__(): # TODO detect for TypeError: duplicate base class str, # e.g. `class X(str, str): pass` try: @@ -426,24 +451,24 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): pass else: add(cls) - for cls_new in mro_method(evaluator): + for cls_new in mro_method(): add(cls_new) return tuple(mro) @memoize_default(default=()) - def py__bases__(self, evaluator): + def py__bases__(self): arglist = self.base.get_super_arglist() if arglist: args = param.Arguments(self._evaluator, arglist) return list(chain.from_iterable(args.eval_args())) else: - return [compiled.object_obj] + return [compiled.create(self._evaluator, object)] - def py__call__(self, evaluator, params): - return [Instance(evaluator, self, params)] + def py__call__(self, params): + return set([Instance(self._evaluator, self, params)]) - def py__getattribute__(self, name): - return self._evaluator.find_types(self, name) + def py__class__(self): + return compiled.create(self._evaluator, type) @property def params(self): @@ -453,7 +478,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): if search_global: yield self.names_dict else: - for scope in self.py__mro__(self._evaluator): + for scope in self.py__mro__(): if isinstance(scope, compiled.CompiledObject): yield scope.names_dicts(False, is_instance)[0] else: @@ -463,7 +488,7 @@ class Class(use_metaclass(CachedMetaClass, Wrapper)): return True def get_subscope_by_name(self, name): - for s in self.py__mro__(self._evaluator): + for s in self.py__mro__(): for sub in reversed(s.subscopes): if sub.name.value == name: return sub @@ -527,8 +552,10 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)): # Create param array. if isinstance(f, Function): old_func = f # TODO this is just hacky. change. - else: + elif f.type == 'funcdef': old_func = Function(self._evaluator, f, is_decorated=True) + else: + old_func = f wrappers = self._evaluator.execute_evaluated(decorator, old_func) if not len(wrappers): @@ -538,7 +565,7 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)): # TODO resolve issue with multiple wrappers -> multiple types debug.warning('multiple wrappers found %s %s', self.base_func, wrappers) - f = wrappers[0] + f = list(wrappers)[0] if isinstance(f, (Instance, Function)): f.decorates = self @@ -549,15 +576,33 @@ class Function(use_metaclass(CachedMetaClass, Wrapper)): if search_global: yield self.names_dict else: - for names_dict in compiled.magic_function_class.names_dicts(False): + scope = compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS') + for names_dict in scope.names_dicts(False): yield names_dict @Python3Method - def py__call__(self, evaluator, params): + def py__call__(self, params): if self.base.is_generator(): - return [iterable.Generator(evaluator, self, params)] + return set([iterable.Generator(self._evaluator, self, params)]) else: - return FunctionExecution(evaluator, self, params).get_return_types() + return FunctionExecution(self._evaluator, self, params).get_return_types() + + @memoize_default() + def py__annotations__(self): + parser_func = self.base + return_annotation = parser_func.annotation() + if return_annotation: + dct = {'return': return_annotation} + else: + dct = {} + for function_param in parser_func.params: + param_annotation = function_param.annotation() + if param_annotation is not None: + dct[function_param.name.value] = param_annotation + return dct + + def py__class__(self): + return compiled.get_special_object(self._evaluator, 'FUNCTION_CLASS') def __getattr__(self, name): return getattr(self.base_func, name) @@ -588,11 +633,22 @@ class FunctionExecution(Executed): def __init__(self, evaluator, base, *args, **kwargs): super(FunctionExecution, self).__init__(evaluator, base, *args, **kwargs) self._copy_dict = {} - new_func = helpers.deep_ast_copy(base.base_func, self, self._copy_dict) - self.children = new_func.children - self.names_dict = new_func.names_dict + funcdef = base.base_func + if isinstance(funcdef, mixed.MixedObject): + # The extra information in mixed is not needed anymore. We can just + # unpack it and give it the tree object. + funcdef = funcdef.definition - @memoize_default(default=()) + # Just overwrite the old version. We don't need it anymore. + funcdef = helpers.deep_ast_copy(funcdef, new_elements=self._copy_dict) + for child in funcdef.children: + if child.type not in ('operator', 'keyword'): + # Not all nodes are properly copied by deep_ast_copy. + child.parent = self + self.children = funcdef.children + self.names_dict = funcdef.names_dict + + @memoize_default(default=set()) @recursion.execution_recursion_decorator def get_return_types(self, check_yields=False): func = self.base @@ -607,26 +663,87 @@ class FunctionExecution(Executed): # If we do have listeners, that means that there's not a regular # execution ongoing. In this case Jedi is interested in the # inserted params, not in the actual execution of the function. - return [] + return set() if check_yields: - types = [] + types = set() returns = self.yields else: returns = self.returns - types = list(docstrings.find_return_types(self._evaluator, func)) + types = set(docstrings.find_return_types(self._evaluator, func)) + types |= set(pep0484.find_return_types(self._evaluator, func)) for r in returns: check = flow_analysis.break_check(self._evaluator, self, r) if check is flow_analysis.UNREACHABLE: debug.dbg('Return unreachable: %s', r) else: - types += self._evaluator.eval_element(r.children[1]) + if check_yields: + types |= iterable.unite(self._eval_yield(r)) + else: + types |= self._evaluator.eval_element(r.children[1]) if check is flow_analysis.REACHABLE: debug.dbg('Return reachable: %s', r) break return types + def _eval_yield(self, yield_expr): + element = yield_expr.children[1] + if element.type == 'yield_arg': + # It must be a yield from. + yield_from_types = self._evaluator.eval_element(element.children[1]) + for result in iterable.py__iter__(self._evaluator, yield_from_types, element): + yield result + else: + yield self._evaluator.eval_element(element) + + @recursion.execution_recursion_decorator + def get_yield_types(self): + yields = self.yields + stopAt = tree.ForStmt, tree.WhileStmt, FunctionExecution, tree.IfStmt + for_parents = [(x, x.get_parent_until((stopAt))) for x in yields] + + # Calculate if the yields are placed within the same for loop. + yields_order = [] + last_for_stmt = None + for yield_, for_stmt in for_parents: + # For really simple for loops we can predict the order. Otherwise + # we just ignore it. + parent = for_stmt.parent + if parent.type == 'suite': + parent = parent.parent + if for_stmt.type == 'for_stmt' and parent == self \ + and for_stmt.defines_one_name(): # Simplicity for now. + if for_stmt == last_for_stmt: + yields_order[-1][1].append(yield_) + else: + yields_order.append((for_stmt, [yield_])) + elif for_stmt == self: + yields_order.append((None, [yield_])) + else: + yield self.get_return_types(check_yields=True) + return + last_for_stmt = for_stmt + + evaluator = self._evaluator + for for_stmt, yields in yields_order: + if for_stmt is None: + # No for_stmt, just normal yields. + for yield_ in yields: + for result in self._eval_yield(yield_): + yield result + else: + input_node = for_stmt.get_input_node() + for_types = evaluator.eval_element(input_node) + ordered = iterable.py__iter__(evaluator, for_types, input_node) + for index_types in ordered: + dct = {str(for_stmt.children[1]): index_types} + evaluator.predefined_if_name_dict_dict[for_stmt] = dct + for yield_in_same_for_stmt in yields: + for result in self._eval_yield(yield_in_same_for_stmt): + yield result + del evaluator.predefined_if_name_dict_dict[for_stmt] + def names_dicts(self, search_global): yield self.names_dict @@ -646,50 +763,28 @@ class FunctionExecution(Executed): def name_for_position(self, position): return tree.Function.name_for_position(self, position) - def _copy_list(self, lst): - """ - Copies a list attribute of a parser Function. Copying is very - expensive, because it is something like `copy.deepcopy`. However, these - copied objects can be used for the executions, as if they were in the - execution. - """ - objects = [] - for element in lst: - self._scope_copy(element.parent) - copied = helpers.deep_ast_copy(element, self._copy_dict) - objects.append(copied) - return objects - def __getattr__(self, name): if name not in ['start_pos', 'end_pos', 'imports', 'name', 'type']: raise AttributeError('Tried to access %s: %s. Why?' % (name, self)) return getattr(self.base, name) - def _scope_copy(self, scope): - raise NotImplementedError - """ Copies a scope (e.g. `if foo:`) in an execution """ - if scope != self.base.base_func: - # Just make sure the parents been copied. - self._scope_copy(scope.parent) - helpers.deep_ast_copy(scope, self._copy_dict) - @common.safe_property - @memoize_default([]) + @memoize_default() def returns(self): return tree.Scope._search_in_scope(self, tree.ReturnStmt) @common.safe_property - @memoize_default([]) + @memoize_default() def yields(self): return tree.Scope._search_in_scope(self, tree.YieldExpr) @common.safe_property - @memoize_default([]) + @memoize_default() def statements(self): return tree.Scope._search_in_scope(self, tree.ExprStmt) @common.safe_property - @memoize_default([]) + @memoize_default() def subscopes(self): return tree.Scope._search_in_scope(self, tree.Scope) @@ -742,7 +837,8 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)): @memoize_default() def _module_attributes_dict(self): def parent_callback(): - return self._evaluator.execute(compiled.create(self._evaluator, str))[0] + # Create a string type object (without a defined string in it): + return list(self._evaluator.execute(compiled.create(self._evaluator, str)))[0] names = ['__file__', '__package__', '__doc__', '__name__'] # All the additional module attributes are strings. @@ -757,7 +853,8 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)): def _get_init_directory(self): for suffix, _, _ in imp.get_suffixes(): ending = '__init__' + suffix - if self.py__file__().endswith(ending): + py__file__ = self.py__file__() + if py__file__ is not None and py__file__.endswith(ending): # Remove the ending, including the separator. return self.py__file__()[:-len(ending) - 1] return None @@ -850,6 +947,9 @@ class ModuleWrapper(use_metaclass(CachedMetaClass, tree.Module, Wrapper)): return names + def py__class__(self): + return compiled.get_special_object(self._evaluator, 'MODULE_CLASS') + def __getattr__(self, name): return getattr(self._module, name) diff --git a/jedi/evaluate/stdlib.py b/jedi/evaluate/stdlib.py index 20ff5d51..f0c4346d 100644 --- a/jedi/evaluate/stdlib.py +++ b/jedi/evaluate/stdlib.py @@ -5,32 +5,37 @@ understand them with Jedi. To add a new implementation, create a function and add it to the ``_implemented`` dict at the bottom of this module. +Note that this module exists only to implement very specific functionality in +the standard library. The usual way to understand the standard library is the +compiled module that returns the types for C-builtins. """ import collections import re from jedi._compatibility import unicode +from jedi.common import unite from jedi.evaluate import compiled from jedi.evaluate import representation as er from jedi.evaluate import iterable -from jedi.parser import Parser +from jedi.parser import ParserWithRecovery from jedi.parser import tree from jedi import debug from jedi.evaluate import precedence from jedi.evaluate import param +from jedi.evaluate import analysis class NotInStdLib(LookupError): pass -def execute(evaluator, obj, params): +def execute(evaluator, obj, arguments): try: obj_name = str(obj.name) except AttributeError: pass else: - if obj.parent == compiled.builtin: + if obj.parent == evaluator.BUILTINS: module_name = 'builtins' elif isinstance(obj.parent, tree.Module): module_name = str(obj.parent.name) @@ -39,22 +44,24 @@ def execute(evaluator, obj, params): # for now we just support builtin functions. try: - return _implemented[module_name][obj_name](evaluator, obj, params) + func = _implemented[module_name][obj_name] except KeyError: pass + else: + return func(evaluator, obj, arguments) raise NotInStdLib() -def _follow_param(evaluator, params, index): +def _follow_param(evaluator, arguments, index): try: - key, values = list(params.unpack())[index] + key, values = list(arguments.unpack())[index] except IndexError: - return [] + return set() else: - return iterable.unite(evaluator.eval_element(v) for v in values) + return unite(evaluator.eval_element(v) for v in values) -def argument_clinic(string, want_obj=False, want_scope=False): +def argument_clinic(string, want_obj=False, want_scope=False, want_arguments=False): """ Works like Argument Clinic (PEP 436), to validate function params. """ @@ -77,17 +84,22 @@ def argument_clinic(string, want_obj=False, want_scope=False): def f(func): def wrapper(evaluator, obj, arguments): + debug.dbg('builtin start %s' % obj, color='MAGENTA') try: lst = list(arguments.eval_argument_clinic(clinic_args)) except ValueError: - return [] + return set() else: kwargs = {} if want_scope: kwargs['scope'] = arguments.scope() if want_obj: kwargs['obj'] = obj + if want_arguments: + kwargs['arguments'] = arguments return func(evaluator, *lst, **kwargs) + finally: + debug.dbg('builtin end', color='MAGENTA') return wrapper return f @@ -95,7 +107,6 @@ def argument_clinic(string, want_obj=False, want_scope=False): @argument_clinic('object, name[, default], /') def builtins_getattr(evaluator, objects, names, defaults=None): - types = [] # follow the first param for obj in objects: if not isinstance(obj, (er.Instance, er.Class, tree.Module, compiled.CompiledObject)): @@ -108,16 +119,16 @@ def builtins_getattr(evaluator, objects, names, defaults=None): else: debug.warning('getattr called without str') continue - return types + return set() @argument_clinic('object[, bases, dict], /') def builtins_type(evaluator, objects, bases, dicts): if bases or dicts: - # metaclass... maybe someday... - return [] + # It's a type creation... maybe someday... + return set() else: - return [o.base for o in objects if isinstance(o, er.Instance)] + return set([o.py__class__() for o in objects]) class SuperInstance(er.Instance): @@ -140,17 +151,21 @@ def builtins_super(evaluator, types, objects, scope): cls = er.Class(evaluator, cls) elif isinstance(cls, er.Instance): cls = cls.base - su = cls.py__bases__(evaluator) + su = cls.py__bases__() if su: return evaluator.execute(su[0]) - return [] + return set() -@argument_clinic('sequence, /', want_obj=True) -def builtins_reversed(evaluator, sequences, obj): - # Unpack the iterator values - objects = tuple(iterable.get_iterator_types(sequences)) - rev = [iterable.AlreadyEvaluated([o]) for o in reversed(objects)] +@argument_clinic('sequence, /', want_obj=True, want_arguments=True) +def builtins_reversed(evaluator, sequences, obj, arguments): + # While we could do without this variable (just by using sequences), we + # want static analysis to work well. Therefore we need to generated the + # values again. + first_arg = next(arguments.as_tuple())[0] + ordered = list(iterable.py__iter__(evaluator, sequences, first_arg)) + + rev = [iterable.AlreadyEvaluated(o) for o in reversed(ordered)] # Repack iterator values and then run it the normal way. This is # necessary, because `reversed` is a function and autocompletion # would fail in certain cases like `reversed(x).__iter__` if we @@ -158,35 +173,43 @@ def builtins_reversed(evaluator, sequences, obj): rev = iterable.AlreadyEvaluated( [iterable.FakeSequence(evaluator, rev, 'list')] ) - return [er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))] + return set([er.Instance(evaluator, obj, param.Arguments(evaluator, [rev]))]) -@argument_clinic('obj, type, /') -def builtins_isinstance(evaluator, objects, types): +@argument_clinic('obj, type, /', want_arguments=True) +def builtins_isinstance(evaluator, objects, types, arguments): bool_results = set([]) for o in objects: try: - mro_func = o.py__class__(evaluator).py__mro__ + mro_func = o.py__class__().py__mro__ except AttributeError: # This is temporary. Everything should have a class attribute in # Python?! Maybe we'll leave it here, because some numpy objects or # whatever might not. - return [compiled.true_obj, compiled.false_obj] + return set([compiled.create(True), compiled.create(False)]) - mro = mro_func(evaluator) + mro = mro_func() for cls_or_tup in types: if cls_or_tup.is_class(): bool_results.add(cls_or_tup in mro) - else: + elif str(cls_or_tup.name) == 'tuple' \ + and cls_or_tup.get_parent_scope() == evaluator.BUILTINS: # Check for tuples. - classes = iterable.get_iterator_types([cls_or_tup]) + classes = unite(cls_or_tup.py__iter__()) bool_results.add(any(cls in mro for cls in classes)) + else: + _, nodes = list(arguments.unpack())[1] + for node in nodes: + message = 'TypeError: isinstance() arg 2 must be a ' \ + 'class, type, or tuple of classes and types, ' \ + 'not %s.' % cls_or_tup + analysis.add(evaluator, 'type-error-isinstance', node, message) - return [compiled.keyword_from_value(x) for x in bool_results] + return set(compiled.create(evaluator, x) for x in bool_results) -def collections_namedtuple(evaluator, obj, params): +def collections_namedtuple(evaluator, obj, arguments): """ Implementation of the namedtuple function. @@ -198,20 +221,21 @@ def collections_namedtuple(evaluator, obj, params): """ # Namedtuples are not supported on Python 2.6 if not hasattr(collections, '_class_template'): - return [] + return set() # Process arguments - name = _follow_param(evaluator, params, 0)[0].obj - _fields = _follow_param(evaluator, params, 1)[0] + # TODO here we only use one of the types, we should use all. + name = list(_follow_param(evaluator, arguments, 0))[0].obj + _fields = list(_follow_param(evaluator, arguments, 1))[0] if isinstance(_fields, compiled.CompiledObject): fields = _fields.obj.replace(',', ' ').split() elif isinstance(_fields, iterable.Array): try: - fields = [v.obj for v in _fields.values()] + fields = [v.obj for v in unite(_fields.py__iter__())] except AttributeError: - return [] + return set() else: - return [] + return set() # Build source source = collections._class_template.format( @@ -225,8 +249,8 @@ def collections_namedtuple(evaluator, obj, params): ) # Parse source - generated_class = Parser(evaluator.grammar, unicode(source)).module.subscopes[0] - return [er.Class(evaluator, generated_class)] + generated_class = ParserWithRecovery(evaluator.grammar, unicode(source)).module.subscopes[0] + return set([er.Class(evaluator, generated_class)]) @argument_clinic('first, /') @@ -247,8 +271,8 @@ _implemented = { 'deepcopy': _return_first_param, }, 'json': { - 'load': lambda *args: [], - 'loads': lambda *args: [], + 'load': lambda *args: set(), + 'loads': lambda *args: set(), }, 'collections': { 'namedtuple': collections_namedtuple, diff --git a/jedi/evaluate/sys_path.py b/jedi/evaluate/sys_path.py index 50e34b9d..658f66a8 100644 --- a/jedi/evaluate/sys_path.py +++ b/jedi/evaluate/sys_path.py @@ -5,11 +5,11 @@ from jedi.evaluate.site import addsitedir from jedi._compatibility import exec_function, unicode from jedi.parser import tree -from jedi.parser import Parser +from jedi.parser import ParserWithRecovery from jedi.evaluate.cache import memoize_default from jedi import debug from jedi import common -from jedi import cache +from jedi.parser.utils import load_parser, save_parser def get_venv_path(venv): @@ -99,7 +99,8 @@ def _paths_from_assignment(evaluator, expr_stmt): for assignee, operator in zip(expr_stmt.children[::2], expr_stmt.children[1::2]): try: assert operator in ['=', '+='] - assert tree.is_node(assignee, 'power') and len(assignee.children) > 1 + assert tree.is_node(assignee, 'power', 'atom_expr') and \ + len(assignee.children) > 1 c = assignee.children assert c[0].type == 'name' and c[0].value == 'sys' trailer = c[1] @@ -118,11 +119,13 @@ def _paths_from_assignment(evaluator, expr_stmt): except AssertionError: continue - from jedi.evaluate.iterable import get_iterator_types + from jedi.evaluate.iterable import py__iter__ from jedi.evaluate.precedence import is_string - for val in get_iterator_types(evaluator.eval_statement(expr_stmt)): - if is_string(val): - yield val.obj + types = evaluator.eval_element(expr_stmt) + for types in py__iter__(evaluator, types, expr_stmt): + for typ in types: + if is_string(typ): + yield typ.obj def _paths_from_list_modifications(module_path, trailer1, trailer2): @@ -150,7 +153,7 @@ def _check_module(evaluator, module): def get_sys_path_powers(names): for name in names: power = name.parent.parent - if tree.is_node(power, 'power'): + if tree.is_node(power, 'power', 'atom_expr'): c = power.children if isinstance(c[0], tree.Name) and c[0].value == 'sys' \ and tree.is_node(c[1], 'trailer'): @@ -183,6 +186,7 @@ def sys_path_with_modifications(evaluator, module): return list(evaluator.sys_path) curdir = os.path.abspath(os.curdir) + #TODO why do we need a chdir? with common.ignored(OSError): os.chdir(os.path.dirname(module.path)) @@ -207,11 +211,11 @@ def _get_paths_from_buildout_script(evaluator, buildout_script): debug.dbg('Error trying to read buildout_script: %s', buildout_script) return - p = Parser(evaluator.grammar, source, buildout_script) - cache.save_parser(buildout_script, p) + p = ParserWithRecovery(evaluator.grammar, source, buildout_script) + save_parser(buildout_script, p) return p.module - cached = cache.load_parser(buildout_script) + cached = load_parser(buildout_script) module = cached and cached.module or load(buildout_script) if not module: return @@ -271,8 +275,8 @@ def _get_buildout_scripts(module_path): firstline = f.readline() if firstline.startswith('#!') and 'python' in firstline: extra_module_paths.append(filepath) - except IOError as e: - # either permission error or race cond. because file got deleted + except (UnicodeDecodeError, IOError) as e: + # Probably a binary file; permission error or race cond. because file got deleted # ignore debug.warning(unicode(e)) continue diff --git a/jedi/parser/__init__.py b/jedi/parser/__init__.py index 3dc982bc..8fb915a5 100644 --- a/jedi/parser/__init__.py +++ b/jedi/parser/__init__.py @@ -20,9 +20,8 @@ import re from jedi.parser import tree as pt from jedi.parser import tokenize -from jedi.parser import token from jedi.parser.token import (DEDENT, INDENT, ENDMARKER, NEWLINE, NUMBER, - STRING, OP, ERRORTOKEN) + STRING) from jedi.parser.pgen2.pgen import generate_grammar from jedi.parser.pgen2.parse import PgenParser @@ -35,45 +34,30 @@ STATEMENT_KEYWORDS = 'assert', 'del', 'global', 'nonlocal', 'raise', \ _loaded_grammars = {} -def load_grammar(file='grammar3.4'): +class ParseError(Exception): + """ + Signals you that the code you fed the Parser was not correct Python code. + """ + + +def load_grammar(version='3.4'): # For now we only support two different Python syntax versions: The latest # Python 3 and Python 2. This may change. - if file.startswith('grammar3'): - file = 'grammar3.4' - else: - file = 'grammar2.7' + if version in ('3.2', '3.3'): + version = '3.4' + elif version == '2.6': + version = '2.7' + + file = 'grammar' + version + '.txt' global _loaded_grammars - path = os.path.join(os.path.dirname(__file__), file) + '.txt' + path = os.path.join(os.path.dirname(__file__), file) try: return _loaded_grammars[path] except KeyError: return _loaded_grammars.setdefault(path, generate_grammar(path)) -class ErrorStatement(object): - def __init__(self, stack, next_token, position_modifier, next_start_pos): - self.stack = stack - self._position_modifier = position_modifier - self.next_token = next_token - self._next_start_pos = next_start_pos - - @property - def next_start_pos(self): - s = self._next_start_pos - return s[0] + self._position_modifier.line, s[1] - - @property - def first_pos(self): - first_type, nodes = self.stack[0] - return nodes[0].start_pos - - @property - def first_type(self): - first_type, nodes = self.stack[0] - return first_type - - class ParserSyntaxError(object): def __init__(self, message, position): self.message = message @@ -81,91 +65,96 @@ class ParserSyntaxError(object): class Parser(object): - """ - This class is used to parse a Python file, it then divides them into a - class structure of different scopes. + AST_MAPPING = { + 'expr_stmt': pt.ExprStmt, + 'classdef': pt.Class, + 'funcdef': pt.Function, + 'file_input': pt.Module, + 'import_name': pt.ImportName, + 'import_from': pt.ImportFrom, + 'break_stmt': pt.KeywordStatement, + 'continue_stmt': pt.KeywordStatement, + 'return_stmt': pt.ReturnStmt, + 'raise_stmt': pt.KeywordStatement, + 'yield_expr': pt.YieldExpr, + 'del_stmt': pt.KeywordStatement, + 'pass_stmt': pt.KeywordStatement, + 'global_stmt': pt.GlobalStmt, + 'nonlocal_stmt': pt.KeywordStatement, + 'print_stmt': pt.KeywordStatement, + 'assert_stmt': pt.AssertStmt, + 'if_stmt': pt.IfStmt, + 'with_stmt': pt.WithStmt, + 'for_stmt': pt.ForStmt, + 'while_stmt': pt.WhileStmt, + 'try_stmt': pt.TryStmt, + 'comp_for': pt.CompFor, + 'decorator': pt.Decorator, + 'lambdef': pt.Lambda, + 'old_lambdef': pt.Lambda, + 'lambdef_nocond': pt.Lambda, + } - :param grammar: The grammar object of pgen2. Loaded by load_grammar. - :param source: The codebase for the parser. Must be unicode. - :param module_path: The path of the module in the file system, may be None. - :type module_path: str - :param top_module: Use this module as a parent instead of `self.module`. - """ - def __init__(self, grammar, source, module_path=None, tokenizer=None): - self._ast_mapping = { - 'expr_stmt': pt.ExprStmt, - 'classdef': pt.Class, - 'funcdef': pt.Function, - 'file_input': pt.Module, - 'import_name': pt.ImportName, - 'import_from': pt.ImportFrom, - 'break_stmt': pt.KeywordStatement, - 'continue_stmt': pt.KeywordStatement, - 'return_stmt': pt.ReturnStmt, - 'raise_stmt': pt.KeywordStatement, - 'yield_expr': pt.YieldExpr, - 'del_stmt': pt.KeywordStatement, - 'pass_stmt': pt.KeywordStatement, - 'global_stmt': pt.GlobalStmt, - 'nonlocal_stmt': pt.KeywordStatement, - 'assert_stmt': pt.AssertStmt, - 'if_stmt': pt.IfStmt, - 'with_stmt': pt.WithStmt, - 'for_stmt': pt.ForStmt, - 'while_stmt': pt.WhileStmt, - 'try_stmt': pt.TryStmt, - 'comp_for': pt.CompFor, - 'decorator': pt.Decorator, - 'lambdef': pt.Lambda, - 'old_lambdef': pt.Lambda, - 'lambdef_nocond': pt.Lambda, - } + def __init__(self, grammar, source, start_symbol='file_input', + tokenizer=None, start_parsing=True): + # Todo Remove start_parsing (with False) - self.syntax_errors = [] - - self._global_names = [] - self._omit_dedent_list = [] - self._indent_counter = 0 - self._last_failed_start_pos = (0, 0) - - # TODO do print absolute import detection here. - #try: - # del python_grammar_no_print_statement.keywords["print"] - #except KeyError: - # pass # Doesn't exist in the Python 3 grammar. - - #if self.options["print_function"]: - # python_grammar = pygram.python_grammar_no_print_statement - #else: self._used_names = {} self._scope_names_stack = [{}] - self._error_statement_stacks = [] - - added_newline = False - # The Python grammar needs a newline at the end of each statement. - if not source.endswith('\n'): - source += '\n' - added_newline = True + self._last_failed_start_pos = (0, 0) + self._global_names = [] # For the fast parser. self.position_modifier = pt.PositionModifier() - p = PgenParser(grammar, self.convert_node, self.convert_leaf, - self.error_recovery) - tokenizer = tokenizer or tokenize.source_tokens(source) - self.module = p.parse(self._tokenize(tokenizer)) - if self.module.type != 'file_input': + + self._added_newline = False + # The Python grammar needs a newline at the end of each statement. + if not source.endswith('\n') and start_symbol == 'file_input': + source += '\n' + self._added_newline = True + + self._start_symbol = start_symbol + self._grammar = grammar + + self._parsed = None + + if start_parsing: + if tokenizer is None: + tokenizer = tokenize.source_tokens(source, use_exact_op_types=True) + self.parse(tokenizer) + + def parse(self, tokenizer): + if self._parsed is not None: + return self._parsed + + start_number = self._grammar.symbol2number[self._start_symbol] + pgen_parser = PgenParser( + self._grammar, self.convert_node, self.convert_leaf, + self.error_recovery, start_number + ) + + try: + self._parsed = pgen_parser.parse(tokenizer) + finally: + self.stack = pgen_parser.stack + + if self._start_symbol == 'file_input' != self._parsed.type: # If there's only one statement, we get back a non-module. That's # not what we want, we want a module, so we add it here: - self.module = self.convert_node(grammar, - grammar.symbol2number['file_input'], - [self.module]) + self._parsed = self.convert_node(self._grammar, + self._grammar.symbol2number['file_input'], + [self._parsed]) - if added_newline: + if self._added_newline: self.remove_last_newline() - self.module.used_names = self._used_names - self.module.path = module_path - self.module.global_names = self._global_names - self.module.error_statement_stacks = self._error_statement_stacks + + def get_parsed_node(self): + # TODO rename to get_root_node + return self._parsed + + def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback): + raise ParseError def convert_node(self, grammar, type, children): """ @@ -177,7 +166,7 @@ class Parser(object): """ symbol = grammar.number2symbol[type] try: - new_node = self._ast_mapping[symbol](children) + new_node = Parser.AST_MAPPING[symbol](children) except KeyError: new_node = pt.Node(symbol, children) @@ -206,7 +195,7 @@ class Parser(object): return new_node def convert_leaf(self, grammar, type, value, prefix, start_pos): - #print('leaf', value, pytree.type_repr(type)) + # print('leaf', repr(value), token.tok_name[type]) if type == tokenize.NAME: if value in grammar.keywords: if value in ('def', 'class', 'lambda'): @@ -227,156 +216,51 @@ class Parser(object): return pt.Number(self.position_modifier, value, start_pos, prefix) elif type in (NEWLINE, ENDMARKER): return pt.Whitespace(self.position_modifier, value, start_pos, prefix) + elif type == INDENT: + return pt.Indent(self.position_modifier, value, start_pos, prefix) + elif type == DEDENT: + return pt.Dedent(self.position_modifier, value, start_pos, prefix) else: return pt.Operator(self.position_modifier, value, start_pos, prefix) - def error_recovery(self, grammar, stack, typ, value, start_pos, prefix, - add_token_callback): - """ - This parser is written in a dynamic way, meaning that this parser - allows using different grammars (even non-Python). However, error - recovery is purely written for Python. - """ - def current_suite(stack): - # For now just discard everything that is not a suite or - # file_input, if we detect an error. - for index, (dfa, state, (typ, nodes)) in reversed(list(enumerate(stack))): - # `suite` can sometimes be only simple_stmt, not stmt. - symbol = grammar.number2symbol[typ] - if symbol == 'file_input': - break - elif symbol == 'suite' and len(nodes) > 1: - # suites without an indent in them get discarded. - break - elif symbol == 'simple_stmt' and len(nodes) > 1: - # simple_stmt can just be turned into a Node, if there are - # enough statements. Ignore the rest after that. - break - return index, symbol, nodes - - index, symbol, nodes = current_suite(stack) - if symbol == 'simple_stmt': - index -= 2 - (_, _, (typ, suite_nodes)) = stack[index] - symbol = grammar.number2symbol[typ] - suite_nodes.append(pt.Node(symbol, list(nodes))) - # Remove - nodes[:] = [] - nodes = suite_nodes - stack[index] - - #print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index) - self._stack_removal(grammar, stack, index + 1, value, start_pos) - if typ == INDENT: - # For every deleted INDENT we have to delete a DEDENT as well. - # Otherwise the parser will get into trouble and DEDENT too early. - self._omit_dedent_list.append(self._indent_counter) - - if value in ('import', 'from', 'class', 'def', 'try', 'while', 'return'): - # Those can always be new statements. - add_token_callback(typ, value, prefix, start_pos) - elif typ == DEDENT and symbol == 'suite': - # Close the current suite, with DEDENT. - # Note that this may cause some suites to not contain any - # statements at all. This is contrary to valid Python syntax. We - # keep incomplete suites in Jedi to be able to complete param names - # or `with ... as foo` names. If we want to use this parser for - # syntax checks, we have to check in a separate turn if suites - # contain statements or not. However, a second check is necessary - # anyway (compile.c does that for Python), because Python's grammar - # doesn't stop you from defining `continue` in a module, etc. - add_token_callback(typ, value, prefix, start_pos) - - def _stack_removal(self, grammar, stack, start_index, value, start_pos): - def clear_names(children): - for c in children: - try: - clear_names(c.children) - except AttributeError: - if isinstance(c, pt.Name): - try: - self._scope_names_stack[-1][c.value].remove(c) - self._used_names[c.value].remove(c) - except ValueError: - pass # This may happen with CompFor. - - for dfa, state, node in stack[start_index:]: - clear_names(children=node[1]) - - failed_stack = [] - found = False - for dfa, state, (typ, nodes) in stack[start_index:]: - if nodes: - found = True - if found: - symbol = grammar.number2symbol[typ] - failed_stack.append((symbol, nodes)) - if nodes and nodes[0] in ('def', 'class', 'lambda'): - self._scope_names_stack.pop() - if failed_stack: - err = ErrorStatement(failed_stack, value, self.position_modifier, start_pos) - self._error_statement_stacks.append(err) - - self._last_failed_start_pos = start_pos - - stack[start_index:] = [] - - def _tokenize(self, tokenizer): - for typ, value, start_pos, prefix in tokenizer: - #print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix)) - if typ == DEDENT: - # We need to count indents, because if we just omit any DEDENT, - # we might omit them in the wrong place. - o = self._omit_dedent_list - if o and o[-1] == self._indent_counter: - o.pop() - continue - - self._indent_counter -= 1 - elif typ == INDENT: - self._indent_counter += 1 - elif typ == ERRORTOKEN: - self._add_syntax_error('Strange token', start_pos) - continue - - if typ == OP: - typ = token.opmap[value] - yield typ, value, prefix, start_pos - - def _add_syntax_error(self, message, position): - self.syntax_errors.append(ParserSyntaxError(message, position)) - - def __repr__(self): - return "<%s: %s>" % (type(self).__name__, self.module) - def remove_last_newline(self): """ In all of this we need to work with _start_pos, because if we worked with start_pos, we would need to check the position_modifier as well (which is accounted for in the start_pos property). """ - endmarker = self.module.children[-1] + endmarker = self._parsed.children[-1] # The newline is either in the endmarker as a prefix or the previous # leaf as a newline token. - if endmarker.prefix.endswith('\n'): - endmarker.prefix = endmarker.prefix[:-1] - last_line = re.sub('.*\n', '', endmarker.prefix) - endmarker._start_pos = endmarker._start_pos[0] - 1, len(last_line) + prefix = endmarker.prefix + if prefix.endswith('\n'): + endmarker.prefix = prefix = prefix[:-1] + last_end = 0 + if '\n' not in prefix: + # Basically if the last line doesn't end with a newline. we + # have to add the previous line's end_position. + try: + last_end = endmarker.get_previous_leaf().end_pos[1] + except IndexError: + pass + last_line = re.sub('.*\n', '', prefix) + endmarker._start_pos = endmarker._start_pos[0] - 1, last_end + len(last_line) else: try: - newline = endmarker.get_previous() + newline = endmarker.get_previous_leaf() except IndexError: return # This means that the parser is empty. while True: if newline.value == '': # Must be a DEDENT, just continue. try: - newline = newline.get_previous() + newline = newline.get_previous_leaf() except IndexError: # If there's a statement that fails to be parsed, there # will be no previous leaf. So just ignore it. break elif newline.value != '\n': + # TODO REMOVE, error recovery was simplified. # This may happen if error correction strikes and removes # a whole statement including '\n'. break @@ -391,3 +275,127 @@ class Parser(object): else: endmarker._start_pos = newline._start_pos break + + +class ParserWithRecovery(Parser): + """ + This class is used to parse a Python file, it then divides them into a + class structure of different scopes. + + :param grammar: The grammar object of pgen2. Loaded by load_grammar. + :param source: The codebase for the parser. Must be unicode. + :param module_path: The path of the module in the file system, may be None. + :type module_path: str + """ + def __init__(self, grammar, source, module_path=None, tokenizer=None): + self.syntax_errors = [] + + self._omit_dedent_list = [] + self._indent_counter = 0 + + # TODO do print absolute import detection here. + # try: + # del python_grammar_no_print_statement.keywords["print"] + # except KeyError: + # pass # Doesn't exist in the Python 3 grammar. + + # if self.options["print_function"]: + # python_grammar = pygram.python_grammar_no_print_statement + # else: + super(ParserWithRecovery, self).__init__(grammar, source, tokenizer=tokenizer) + + self.module = self._parsed + self.module.used_names = self._used_names + self.module.path = module_path + self.module.global_names = self._global_names + + def parse(self, tokenizer): + return super(ParserWithRecovery, self).parse(self._tokenize(self._tokenize(tokenizer))) + + def error_recovery(self, grammar, stack, arcs, typ, value, start_pos, prefix, + add_token_callback): + """ + This parser is written in a dynamic way, meaning that this parser + allows using different grammars (even non-Python). However, error + recovery is purely written for Python. + """ + def current_suite(stack): + # For now just discard everything that is not a suite or + # file_input, if we detect an error. + for index, (dfa, state, (type_, nodes)) in reversed(list(enumerate(stack))): + # `suite` can sometimes be only simple_stmt, not stmt. + symbol = grammar.number2symbol[type_] + if symbol == 'file_input': + break + elif symbol == 'suite' and len(nodes) > 1: + # suites without an indent in them get discarded. + break + elif symbol == 'simple_stmt' and len(nodes) > 1: + # simple_stmt can just be turned into a Node, if there are + # enough statements. Ignore the rest after that. + break + return index, symbol, nodes + + index, symbol, nodes = current_suite(stack) + if symbol == 'simple_stmt': + index -= 2 + (_, _, (type_, suite_nodes)) = stack[index] + symbol = grammar.number2symbol[type_] + suite_nodes.append(pt.Node(symbol, list(nodes))) + # Remove + nodes[:] = [] + nodes = suite_nodes + stack[index] + + # print('err', token.tok_name[typ], repr(value), start_pos, len(stack), index) + if self._stack_removal(grammar, stack, arcs, index + 1, value, start_pos): + add_token_callback(typ, value, start_pos, prefix) + else: + if typ == INDENT: + # For every deleted INDENT we have to delete a DEDENT as well. + # Otherwise the parser will get into trouble and DEDENT too early. + self._omit_dedent_list.append(self._indent_counter) + else: + error_leaf = pt.ErrorLeaf(self.position_modifier, typ, value, start_pos, prefix) + stack[-1][2][1].append(error_leaf) + + def _stack_removal(self, grammar, stack, arcs, start_index, value, start_pos): + failed_stack = [] + found = False + all_nodes = [] + for dfa, state, (typ, nodes) in stack[start_index:]: + if nodes: + found = True + if found: + symbol = grammar.number2symbol[typ] + failed_stack.append((symbol, nodes)) + all_nodes += nodes + if nodes and nodes[0] in ('def', 'class', 'lambda'): + self._scope_names_stack.pop() + if failed_stack: + stack[start_index - 1][2][1].append(pt.ErrorNode(all_nodes)) + + self._last_failed_start_pos = start_pos + + stack[start_index:] = [] + return failed_stack + + def _tokenize(self, tokenizer): + for typ, value, start_pos, prefix in tokenizer: + # print(tokenize.tok_name[typ], repr(value), start_pos, repr(prefix)) + if typ == DEDENT: + # We need to count indents, because if we just omit any DEDENT, + # we might omit them in the wrong place. + o = self._omit_dedent_list + if o and o[-1] == self._indent_counter: + o.pop() + continue + + self._indent_counter -= 1 + elif typ == INDENT: + self._indent_counter += 1 + + yield typ, value, start_pos, prefix + + def __repr__(self): + return "<%s: %s>" % (type(self).__name__, self.module) diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index 35bb8555..520b9d81 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -8,9 +8,9 @@ from itertools import chain from jedi._compatibility import use_metaclass from jedi import settings -from jedi.parser import Parser +from jedi.parser import ParserWithRecovery from jedi.parser import tree -from jedi import cache +from jedi.parser.utils import underscore_memoization, parser_cache from jedi import debug from jedi.parser.tokenize import (source_tokens, NEWLINE, ENDMARKER, INDENT, DEDENT) @@ -36,7 +36,7 @@ class FastModule(tree.Module): pass # It was never used. @property - @cache.underscore_memoization + @underscore_memoization def used_names(self): return MergedNamesDict([m.used_names for m in self.modules]) @@ -45,21 +45,22 @@ class FastModule(tree.Module): return [name for m in self.modules for name in m.global_names] @property - def error_statement_stacks(self): - return [e for m in self.modules for e in m.error_statement_stacks] + def error_statements(self): + return [e for m in self.modules for e in m.error_statements] def __repr__(self): return "" % (type(self).__name__, self.name, self.start_pos[0], self.end_pos[0]) - # To avoid issues with with the `parser.Parser`, we need setters that do - # nothing, because if pickle comes along and sets those values. + # To avoid issues with with the `parser.ParserWithRecovery`, we need + # setters that do nothing, because if pickle comes along and sets those + # values. @global_names.setter def global_names(self, value): pass - @error_statement_stacks.setter - def error_statement_stacks(self, value): + @error_statements.setter + def error_statements(self, value): pass @used_names.setter @@ -99,10 +100,10 @@ class CachedFastParser(type): """ This is a metaclass for caching `FastParser`. """ def __call__(self, grammar, source, module_path=None): if not settings.fast_parser: - return Parser(grammar, source, module_path) + return ParserWithRecovery(grammar, source, module_path) - pi = cache.parser_cache.get(module_path, None) - if pi is None or isinstance(pi.parser, Parser): + pi = parser_cache.get(module_path, None) + if pi is None or isinstance(pi.parser, ParserWithRecovery): p = super(CachedFastParser, self).__call__(grammar, source, module_path) else: p = pi.parser # pi is a `cache.ParserCacheItem` @@ -123,14 +124,21 @@ class ParserNode(object): try: # With fast_parser we have either 1 subscope or only statements. self._content_scope = parser.module.subscopes[0] + # A parsed node's content will be in the first indent, because + # everything that's parsed is within this subscope. + self._is_class_or_def = True except IndexError: self._content_scope = parser.module + self._is_class_or_def = False else: self._rewrite_last_newline() # We need to be able to reset the original children of a parser. self._old_children = list(self._content_scope.children) + def is_root_node(self): + return self.parent is None + def _rewrite_last_newline(self): """ The ENDMARKER can contain a newline in the prefix. However this prefix @@ -181,25 +189,30 @@ class ParserNode(object): dcts.insert(0, self._content_scope.names_dict) self._content_scope.names_dict = MergedNamesDict(dcts) - def parent_until_indent(self, indent=None): - if (indent is None or self._indent >= indent) and self.parent is not None: - self.close() - return self.parent.parent_until_indent(indent) - return self - @property def _indent(self): - if not self.parent: + if self.is_root_node(): return 0 return self.parser.module.children[0].start_pos[1] - def add_node(self, node, line_offset): - """Adding a node means adding a node that was already added earlier""" + def add_node(self, node, start_line, indent): + """ + Adding a node means adding a node that was either just parsed or one + that can be reused. + """ + # Content that is not a subscope can never be part of the current node, + # because it's basically a sister node, that sits next to it and not + # within it. + if (self._indent >= indent or not self._is_class_or_def) and \ + not self.is_root_node(): + self.close() + return self.parent.add_node(node, start_line, indent) + # Changing the line offsets is very important, because if they don't # fit, all the start_pos values will be wrong. m = node.parser.module - node.parser.position_modifier.line = line_offset + node.parser.position_modifier.line = start_line - 1 self._fast_module.modules.append(m) node.parent = self @@ -223,7 +236,7 @@ class ParserNode(object): for y in n.all_sub_nodes(): yield y - @cache.underscore_memoization # Should only happen once! + @underscore_memoization # Should only happen once! def remove_last_newline(self): self.parser.remove_last_newline() @@ -244,11 +257,15 @@ class FastParser(use_metaclass(CachedFastParser)): def _reset_caches(self): self.module = FastModule(self.module_path) - self.current_node = ParserNode(self.module, self, '') + self.root_node = self.current_node = ParserNode(self.module, self, '') + + def get_parsed_node(self): + return self.module def update(self, source): - # For testing purposes: It is important that the number of parsers used - # can be minimized. With these variables we can test against that. + # Variables for testing purposes: It is important that the number of + # parsers used can be minimized. With these variables we can test + # against that. self.number_parsers_used = 0 self.number_of_splits = 0 self.number_of_misses = 0 @@ -312,13 +329,13 @@ class FastParser(use_metaclass(CachedFastParser)): current_lines.append(l) # Just ignore comments and blank lines continue - if new_indent: + if new_indent and not parentheses_level: if indent > indent_list[-2]: # Set the actual indent, not just the random old indent + 1. indent_list[-1] = indent new_indent = False - while indent <= indent_list[-2]: # -> dedent + while indent < indent_list[-1]: # -> dedent indent_list.pop() # This automatically resets the flow_indent if there was a # dedent or a flow just on one line (with one simple_stmt). @@ -348,10 +365,13 @@ class FastParser(use_metaclass(CachedFastParser)): is_decorator = False parentheses_level = \ - max(0, (l.count('(') + l.count('[') + l.count('{') - - l.count(')') - l.count(']') - l.count('}'))) + max(0, (l.count('(') + l.count('[') + l.count('{') - + l.count(')') - l.count(']') - l.count('}'))) current_lines.append(l) + + if previous_line is not None: + current_lines.append(previous_line) if current_lines: yield gen_part() @@ -366,41 +386,44 @@ class FastParser(use_metaclass(CachedFastParser)): source += '\n' added_newline = True - next_line_offset = line_offset = 0 + next_code_part_end_line = code_part_end_line = 1 start = 0 - nodes = list(self.current_node.all_sub_nodes()) + nodes = list(self.root_node.all_sub_nodes()) # Now we can reset the node, because we have all the old nodes. - self.current_node.reset_node() + self.root_node.reset_node() + self.current_node = self.root_node last_end_line = 1 for code_part in self._split_parts(source): - next_line_offset += code_part.count('\n') + next_code_part_end_line += code_part.count('\n') # If the last code part parsed isn't equal to the current end_pos, # we know that the parser went further (`def` start in a # docstring). So just parse the next part. - if line_offset + 1 == last_end_line: - self.current_node = self._get_node(code_part, source[start:], - line_offset, nodes) + if code_part_end_line == last_end_line: + self._parse_part(code_part, source[start:], code_part_end_line, nodes) else: + self.number_of_misses += 1 # Means that some lines where not fully parsed. Parse it now. # This is a very rare case. Should only happens with very # strange code bits. - self.number_of_misses += 1 - while last_end_line < next_line_offset + 1: - line_offset = last_end_line - 1 + while last_end_line < next_code_part_end_line: + code_part_end_line = last_end_line # We could calculate the src in a more complicated way to # make caching here possible as well. However, this is # complicated and error-prone. Since this is not very often # called - just ignore it. - src = ''.join(self._lines[line_offset:]) - self.current_node = self._get_node(code_part, src, - line_offset, nodes) + src = ''.join(self._lines[code_part_end_line - 1:]) + self._parse_part(code_part, src, code_part_end_line, nodes) last_end_line = self.current_node.parser.module.end_pos[0] - + debug.dbg("While parsing %s, starting with line %s wasn't included in split.", + self.module_path, code_part_end_line) + #assert code_part_end_line > last_end_line + # This means that the parser parsed faster than the last given + # `code_part`. debug.dbg('While parsing %s, line %s slowed down the fast parser.', - self.module_path, line_offset + 1) + self.module_path, code_part_end_line) - line_offset = next_line_offset + code_part_end_line = next_code_part_end_line start += len(code_part) last_end_line = self.current_node.parser.module.end_pos[0] @@ -409,39 +432,41 @@ class FastParser(use_metaclass(CachedFastParser)): self.current_node.remove_last_newline() # Now that the for loop is finished, we still want to close all nodes. - self.current_node = self.current_node.parent_until_indent() - self.current_node.close() + node = self.current_node + while node is not None: + node.close() + node = node.parent debug.dbg('Parsed %s, with %s parsers in %s splits.' % (self.module_path, self.number_parsers_used, self.number_of_splits)) - def _get_node(self, source, parser_code, line_offset, nodes): + def _parse_part(self, source, parser_code, code_part_end_line, nodes): """ Side effect: Alters the list of nodes. """ - indent = len(source) - len(source.lstrip('\t ')) - self.current_node = self.current_node.parent_until_indent(indent) - h = hash(source) for index, node in enumerate(nodes): if node.hash == h and node.source == source: node.reset_node() nodes.remove(node) + parser_code = source break else: tokenizer = FastTokenizer(parser_code) self.number_parsers_used += 1 - p = Parser(self._grammar, parser_code, self.module_path, tokenizer=tokenizer) + p = ParserWithRecovery(self._grammar, parser_code, self.module_path, tokenizer=tokenizer) - end = line_offset + p.module.end_pos[0] - used_lines = self._lines[line_offset:end - 1] + end = code_part_end_line - 1 + p.module.end_pos[0] + used_lines = self._lines[code_part_end_line - 1:end - 1] code_part_actually_used = ''.join(used_lines) node = ParserNode(self.module, p, code_part_actually_used) - self.current_node.add_node(node, line_offset) - return node + indent = len(parser_code) - len(parser_code.lstrip('\t ')) + + self.current_node.add_node(node, code_part_end_line, indent) + self.current_node = node class FastTokenizer(object): @@ -450,7 +475,7 @@ class FastTokenizer(object): """ def __init__(self, source): self.source = source - self._gen = source_tokens(source) + self._gen = source_tokens(source, use_exact_op_types=True) self._closed = False # fast parser options @@ -501,19 +526,22 @@ class FastTokenizer(object): self._closed = True return current - if value in ('def', 'class') and self._parentheses_level \ - and re.search(r'\n[ \t]*\Z', prefix): + previous_type = self.previous[0] + if value in ('def', 'class') and self._parentheses_level: # Account for the fact that an open parentheses before a function # will reset the parentheses counter, but new lines before will # still be ignored. So check the prefix. # TODO what about flow parentheses counter resets in the tokenizer? self._parentheses_level = 0 - return self._close() + # We need to simulate a newline before the indent, because the + # open parentheses ignored them. + if re.search('\n\s*', prefix): + previous_type = NEWLINE # Parentheses ignore the indentation rules. The other three stand for # new lines. - if self.previous[0] in (NEWLINE, INDENT, DEDENT) \ + if previous_type in (NEWLINE, INDENT, DEDENT) \ and not self._parentheses_level and typ not in (INDENT, DEDENT): if not self._in_flow: if value in FLOWS: diff --git a/jedi/parser/grammar2.7.txt b/jedi/parser/grammar2.7.txt index b2950143..515dea64 100644 --- a/jedi/parser/grammar2.7.txt +++ b/jedi/parser/grammar2.7.txt @@ -57,7 +57,7 @@ break_stmt: 'break' continue_stmt: 'continue' return_stmt: 'return' [testlist] yield_stmt: yield_expr -raise_stmt: 'raise' [test ['from' test | ',' test [',' test]]] +raise_stmt: 'raise' [test [',' test [',' test]]] import_stmt: import_name | import_from import_name: 'import' dotted_as_names # note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS @@ -68,7 +68,7 @@ dotted_as_name: dotted_name ['as' NAME] import_as_names: import_as_name (',' import_as_name)* [','] dotted_as_names: dotted_as_name (',' dotted_as_name)* dotted_name: NAME ('.' NAME)* -global_stmt: ('global' | 'nonlocal') NAME (',' NAME)* +global_stmt: 'global' NAME (',' NAME)* exec_stmt: 'exec' expr ['in' test [',' test]] assert_stmt: 'assert' test [',' test] diff --git a/jedi/parser/grammar3.5.txt b/jedi/parser/grammar3.5.txt new file mode 100644 index 00000000..96a72718 --- /dev/null +++ b/jedi/parser/grammar3.5.txt @@ -0,0 +1,154 @@ +# Grammar for Python + +# Note: Changing the grammar specified in this file will most likely +# require corresponding changes in the parser module +# (../Modules/parsermodule.c). If you can't make the changes to +# that module yourself, please co-ordinate the required changes +# with someone who can; ask around on python-dev for help. Fred +# Drake will probably be listening there. + +# NOTE WELL: You should also follow all the steps listed at +# https://docs.python.org/devguide/grammar.html + +# Start symbols for the grammar: +# single_input is a single interactive statement; +# file_input is a module or sequence of commands read from an input file; +# eval_input is the input for the eval() functions. +# NB: compound_stmt in single_input is followed by extra NEWLINE! +file_input: (NEWLINE | stmt)* ENDMARKER +single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE +eval_input: testlist NEWLINE* ENDMARKER + +decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE +decorators: decorator+ +decorated: decorators (classdef | funcdef | async_funcdef) + +# NOTE: Reinoud Elhorst, using ASYNC/AWAIT keywords instead of tokens +# skipping python3.5 compatibility, in favour of 3.7 solution +async_funcdef: 'async' funcdef +funcdef: 'def' NAME parameters ['->' test] ':' suite + +parameters: '(' [typedargslist] ')' +typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' + ['*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef]] + | '*' [tfpdef] (',' tfpdef ['=' test])* [',' '**' tfpdef] | '**' tfpdef) +tfpdef: NAME [':' test] +varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' + ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] + | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef) +vfpdef: NAME + +stmt: simple_stmt | compound_stmt +simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE +small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt | + import_stmt | global_stmt | nonlocal_stmt | assert_stmt) +expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | + ('=' (yield_expr|testlist_star_expr))*) +testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [','] +augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | + '<<=' | '>>=' | '**=' | '//=') +# For normal assignments, additional restrictions enforced by the interpreter +del_stmt: 'del' exprlist +pass_stmt: 'pass' +flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt +break_stmt: 'break' +continue_stmt: 'continue' +return_stmt: 'return' [testlist] +yield_stmt: yield_expr +raise_stmt: 'raise' [test ['from' test]] +import_stmt: import_name | import_from +import_name: 'import' dotted_as_names +# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS +import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+) + 'import' ('*' | '(' import_as_names ')' | import_as_names)) +import_as_name: NAME ['as' NAME] +dotted_as_name: dotted_name ['as' NAME] +import_as_names: import_as_name (',' import_as_name)* [','] +dotted_as_names: dotted_as_name (',' dotted_as_name)* +dotted_name: NAME ('.' NAME)* +global_stmt: 'global' NAME (',' NAME)* +nonlocal_stmt: 'nonlocal' NAME (',' NAME)* +assert_stmt: 'assert' test [',' test] + +compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt +async_stmt: 'async' (funcdef | with_stmt | for_stmt) +if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] +while_stmt: 'while' test ':' suite ['else' ':' suite] +for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] +try_stmt: ('try' ':' suite + ((except_clause ':' suite)+ + ['else' ':' suite] + ['finally' ':' suite] | + 'finally' ':' suite)) +with_stmt: 'with' with_item (',' with_item)* ':' suite +with_item: test ['as' expr] +# NB compile.c makes sure that the default except clause is last +except_clause: 'except' [test ['as' NAME]] +# Edit by David Halter: The stmt is now optional. This reflects how Jedi allows +# classes and functions to be empty, which is beneficial for autocompletion. +suite: simple_stmt | NEWLINE INDENT stmt* DEDENT + +test: or_test ['if' or_test 'else' test] | lambdef +test_nocond: or_test | lambdef_nocond +lambdef: 'lambda' [varargslist] ':' test +lambdef_nocond: 'lambda' [varargslist] ':' test_nocond +or_test: and_test ('or' and_test)* +and_test: not_test ('and' not_test)* +not_test: 'not' not_test | comparison +comparison: expr (comp_op expr)* +# <> isn't actually a valid comparison operator in Python. It's here for the +# sake of a __future__ import described in PEP 401 (which really works :-) +comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' +star_expr: '*' expr +expr: xor_expr ('|' xor_expr)* +xor_expr: and_expr ('^' and_expr)* +and_expr: shift_expr ('&' shift_expr)* +shift_expr: arith_expr (('<<'|'>>') arith_expr)* +arith_expr: term (('+'|'-') term)* +term: factor (('*'|'@'|'/'|'%'|'//') factor)* +factor: ('+'|'-'|'~') factor | power +power: atom_expr ['**' factor] +atom_expr: ['await'] atom trailer* +atom: ('(' [yield_expr|testlist_comp] ')' | + '[' [testlist_comp] ']' | + '{' [dictorsetmaker] '}' | + NAME | NUMBER | STRING+ | '...' | 'None' | 'True' | 'False') +testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] ) +trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME +subscriptlist: subscript (',' subscript)* [','] +subscript: test | [test] ':' [test] [sliceop] +sliceop: ':' [test] +exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] +testlist: test (',' test)* [','] +dictorsetmaker: ( ((test ':' test | '**' expr) + (comp_for | (',' (test ':' test | '**' expr))* [','])) | + ((test | star_expr) + (comp_for | (',' (test | star_expr))* [','])) ) + +classdef: 'class' NAME ['(' [arglist] ')'] ':' suite + +arglist: argument (',' argument)* [','] + +# The reason that keywords are test nodes instead of NAME is that using NAME +# results in an ambiguity. ast.c makes sure it's a NAME. +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguements are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test '=' test | + '**' test | + '*' test ) + +comp_iter: comp_for | comp_if +comp_for: 'for' exprlist 'in' or_test [comp_iter] +comp_if: 'if' test_nocond [comp_iter] + +# not used in grammar, but may appear in "node" passed from Parser to Compiler +encoding_decl: NAME + +yield_expr: 'yield' [yield_arg] +yield_arg: 'from' test | testlist diff --git a/jedi/parser/pgen2/parse.py b/jedi/parser/pgen2/parse.py index c8ba70d3..9e74838c 100644 --- a/jedi/parser/pgen2/parse.py +++ b/jedi/parser/pgen2/parse.py @@ -18,8 +18,12 @@ how this parsing engine works. from jedi.parser import tokenize -class ParseError(Exception): - """Exception to signal the parser is stuck.""" +class InternalParseError(Exception): + """ + Exception to signal the parser is stuck and error recovery didn't help. + Basically this shouldn't happen. It's a sign that something is really + wrong. + """ def __init__(self, msg, type, value, start_pos): Exception.__init__(self, "%s: type=%r, value=%r, start_pos=%r" % @@ -30,6 +34,21 @@ class ParseError(Exception): self.start_pos = start_pos +def token_to_ilabel(grammar, type_, value): + # Map from token to label + if type_ == tokenize.NAME: + # Check for reserved words (keywords) + try: + return grammar.keywords[value] + except KeyError: + pass + + try: + return grammar.tokens[type_] + except KeyError: + return None + + class PgenParser(object): """Parser engine. @@ -38,7 +57,7 @@ class PgenParser(object): p = Parser(grammar, [converter]) # create instance p.setup([start]) # prepare for parsing : - if p.addtoken(...): # parse a token; may raise ParseError + if p.addtoken(...): # parse a token break root = p.rootnode # root of abstract syntax tree @@ -53,14 +72,14 @@ class PgenParser(object): Parsing is complete when addtoken() returns True; the root of the abstract syntax tree can then be retrieved from the rootnode - instance variable. When a syntax error occurs, addtoken() raises - the ParseError exception. There is no error recovery; the parser - cannot be used after a syntax error was reported (but it can be - reinitialized by calling setup()). + instance variable. When a syntax error occurs, error_recovery() + is called. There is no error recovery; the parser cannot be used + after a syntax error was reported (but it can be reinitialized by + calling setup()). """ - def __init__(self, grammar, convert_node, convert_leaf, error_recovery): + def __init__(self, grammar, convert_node, convert_leaf, error_recovery, start): """Constructor. The grammar argument is a grammar.Grammar instance; see the @@ -90,8 +109,6 @@ class PgenParser(object): self.convert_node = convert_node self.convert_leaf = convert_leaf - # Prepare for parsing. - start = self.grammar.start # Each stack entry is a tuple: (dfa, state, node). # A node is a tuple: (type, children), # where children is a list of nodes or None @@ -102,29 +119,20 @@ class PgenParser(object): self.error_recovery = error_recovery def parse(self, tokenizer): - for type, value, prefix, start_pos in tokenizer: - if self.addtoken(type, value, prefix, start_pos): + for type_, value, start_pos, prefix in tokenizer: + if self.addtoken(type_, value, start_pos, prefix): break else: # We never broke out -- EOF is too soon -- Unfinished statement. - self.error_recovery(self.grammar, self.stack, type, value, - start_pos, prefix, self.addtoken) - # Add the ENDMARKER again. - if not self.addtoken(type, value, prefix, start_pos): - raise ParseError("incomplete input", type, value, start_pos) + # However, the error recovery might have added the token again, if + # the stack is empty, we're fine. + if self.stack: + raise InternalParseError("incomplete input", type_, value, start_pos) return self.rootnode - def addtoken(self, type, value, prefix, start_pos): + def addtoken(self, type_, value, start_pos, prefix): """Add a token; return True if this is the end of the program.""" - # Map from token to label - if type == tokenize.NAME: - # Check for reserved words (keywords) - try: - ilabel = self.grammar.keywords[value] - except KeyError: - ilabel = self.grammar.tokens[type] - else: - ilabel = self.grammar.tokens[type] + ilabel = token_to_ilabel(self.grammar, type_, value) # Loop until the token is shifted; may raise exceptions while True: @@ -138,7 +146,7 @@ class PgenParser(object): # Look it up in the list of labels assert t < 256 # Shift a token; we're done with it - self.shift(type, value, newstate, prefix, start_pos) + self.shift(type_, value, newstate, prefix, start_pos) # Pop while we are in an accept-only state state = newstate while states[state] == [(0, state)]: @@ -164,36 +172,36 @@ class PgenParser(object): self.pop() if not self.stack: # Done parsing, but another token is input - raise ParseError("too much input", type, value, start_pos) + raise InternalParseError("too much input", type_, value, start_pos) else: - self.error_recovery(self.grammar, self.stack, type, + self.error_recovery(self.grammar, self.stack, arcs, type_, value, start_pos, prefix, self.addtoken) break - def shift(self, type, value, newstate, prefix, start_pos): + def shift(self, type_, value, newstate, prefix, start_pos): """Shift a token. (Internal)""" dfa, state, node = self.stack[-1] - newnode = self.convert_leaf(self.grammar, type, value, prefix, start_pos) + newnode = self.convert_leaf(self.grammar, type_, value, prefix, start_pos) node[-1].append(newnode) self.stack[-1] = (dfa, newstate, node) - def push(self, type, newdfa, newstate): + def push(self, type_, newdfa, newstate): """Push a nonterminal. (Internal)""" dfa, state, node = self.stack[-1] - newnode = (type, []) + newnode = (type_, []) self.stack[-1] = (dfa, newstate, node) self.stack.append((newdfa, 0, newnode)) def pop(self): """Pop a nonterminal. (Internal)""" - popdfa, popstate, (type, children) = self.stack.pop() + popdfa, popstate, (type_, children) = self.stack.pop() # If there's exactly one child, return that child instead of creating a # new node. We still create expr_stmt and file_input though, because a # lot of Jedi depends on its logic. if len(children) == 1: newnode = children[0] else: - newnode = self.convert_node(self.grammar, type, children) + newnode = self.convert_node(self.grammar, type_, children) try: # Equal to: diff --git a/jedi/parser/token.py b/jedi/parser/token.py index e9ab3a62..0cb846da 100644 --- a/jedi/parser/token.py +++ b/jedi/parser/token.py @@ -1,6 +1,6 @@ from __future__ import absolute_import -from jedi._compatibility import is_py3 +from jedi._compatibility import is_py3, is_py35 from token import * @@ -24,6 +24,11 @@ else: tok_name[ELLIPSIS] = 'ELLIPSIS' N_TOKENS += 1 +if not is_py35: + ATEQUAL = N_TOKENS + tok_name[ATEQUAL] = 'ATEQUAL' + N_TOKENS += 1 + # Map from operator to number (since tokenize doesn't do this) @@ -68,6 +73,7 @@ opmap_raw = """\ %= PERCENTEQUAL &= AMPEREQUAL |= VBAREQUAL +@= ATEQUAL ^= CIRCUMFLEXEQUAL <<= LEFTSHIFTEQUAL >>= RIGHTSHIFTEQUAL diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index b3849046..233c6d87 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -13,8 +13,10 @@ from __future__ import absolute_import import string import re +from collections import namedtuple from io import StringIO -from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, + +from jedi.parser.token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, opmap, NAME, OP, ERRORTOKEN, NEWLINE, INDENT, DEDENT) from jedi._compatibility import is_py3 @@ -50,7 +52,10 @@ name = r'\w+' hex_number = r'0[xX][0-9a-fA-F]+' bin_number = r'0[bB][01]+' -oct_number = r'0[oO][0-7]+' +if is_py3: + oct_number = r'0[oO][0-7]+' +else: + oct_number = '0[0-7]+' dec_number = r'(?:0+|[1-9][0-9]*)' int_number = group(hex_number, bin_number, oct_number, dec_number) exponent = r'[eE][-+]?[0-9]+' @@ -76,7 +81,7 @@ triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') # recognized as two instances of =). operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", r"//=?", r"->", - r"[+\-*/%&|^=<>]=?", + r"[+\-*@/%&|^=<>]=?", r"~") bracket = '[][(){}]' @@ -143,18 +148,33 @@ del _compile tabsize = 8 -ALWAYS_BREAK_TOKENS = (';', 'import', 'from', 'class', 'def', 'try', 'except', +# TODO add with? +ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except', 'finally', 'while', 'return') -def source_tokens(source): +class TokenInfo(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])): + def __repr__(self): + annotated_type = tok_name[self.type] + return ('TokenInfo(type=%s, string=%r, start=%r, prefix=%r)' % + self._replace(type=annotated_type)) + + @property + def exact_type(self): + if self.type == OP and self.string in opmap: + return opmap[self.string] + else: + return self.type + + +def source_tokens(source, use_exact_op_types=False): """Generate tokens from a the source code (string).""" - source = source + '\n' # end with \n, because the parser needs it + source = source readline = StringIO(source).readline - return generate_tokens(readline) + return generate_tokens(readline, use_exact_op_types) -def generate_tokens(readline): +def generate_tokens(readline, use_exact_op_types=False): """ A heavily modified Python standard library tokenizer. @@ -165,6 +185,7 @@ def generate_tokens(readline): paren_level = 0 # count parentheses indents = [0] lnum = 0 + max = 0 numchars = '0123456789' contstr = '' contline = None @@ -179,7 +200,7 @@ def generate_tokens(readline): line = readline() # readline returns empty when finished. See StringIO if not line: if contstr: - yield ERRORTOKEN, contstr, contstr_start, prefix + yield TokenInfo(ERRORTOKEN, contstr, contstr_start, prefix) break lnum += 1 @@ -189,7 +210,7 @@ def generate_tokens(readline): endmatch = endprog.match(line) if endmatch: pos = endmatch.end(0) - yield STRING, contstr + line[:pos], contstr_start, prefix + yield TokenInfo(STRING, contstr + line[:pos], contstr_start, prefix) contstr = '' contline = None else: @@ -205,7 +226,7 @@ def generate_tokens(readline): # If a literal starts but doesn't end the whole rest of the # line is an error token. txt = line[pos:] - yield ERRORTOKEN, txt, (lnum, pos), prefix + yield TokenInfo(ERRORTOKEN, txt, (lnum, pos), prefix) pos += 1 continue @@ -218,19 +239,23 @@ def generate_tokens(readline): if new_line and initial not in '\r\n#': new_line = False if paren_level == 0: + i = 0 + while line[i] == '\f': + i += 1 + start -= 1 if start > indents[-1]: - yield INDENT, '', spos, '' + yield TokenInfo(INDENT, '', spos, '') indents.append(start) while start < indents[-1]: - yield DEDENT, '', spos, '' + yield TokenInfo(DEDENT, '', spos, '') indents.pop() if (initial in numchars or # ordinary number (initial == '.' and token != '.' and token != '...')): - yield NUMBER, token, spos, prefix + yield TokenInfo(NUMBER, token, spos, prefix) elif initial in '\r\n': if not new_line and paren_level == 0: - yield NEWLINE, token, spos, prefix + yield TokenInfo(NEWLINE, token, spos, prefix) else: additional_prefix = prefix + token new_line = True @@ -243,7 +268,7 @@ def generate_tokens(readline): if endmatch: # all on one line pos = endmatch.end(0) token = line[start:pos] - yield STRING, token, spos, prefix + yield TokenInfo(STRING, token, spos, prefix) else: contstr_start = (lnum, start) # multiple lines contstr = line[start:] @@ -260,18 +285,18 @@ def generate_tokens(readline): contline = line break else: # ordinary string - yield STRING, token, spos, prefix + yield TokenInfo(STRING, token, spos, prefix) elif is_identifier(initial): # ordinary name if token in ALWAYS_BREAK_TOKENS: paren_level = 0 while True: indent = indents.pop() if indent > start: - yield DEDENT, '', spos, '' + yield TokenInfo(DEDENT, '', spos, '') else: indents.append(indent) break - yield NAME, token, spos, prefix + yield TokenInfo(NAME, token, spos, prefix) elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n'): # continued stmt additional_prefix += prefix + line[start:] break @@ -280,11 +305,25 @@ def generate_tokens(readline): paren_level += 1 elif token in ')]}': paren_level -= 1 - yield OP, token, spos, prefix - end_pos = (lnum, max - 1) + try: + # This check is needed in any case to check if it's a valid + # operator or just some random unicode character. + exact_type = opmap[token] + except KeyError: + exact_type = typ = ERRORTOKEN + if use_exact_op_types: + typ = exact_type + else: + typ = OP + yield TokenInfo(typ, token, spos, prefix) + + if new_line or additional_prefix[-1:] == '\n': + end_pos = lnum + 1, 0 + else: + end_pos = lnum, max # As the last position we just take the maximally possible position. We # remove -1 for the last new line. for indent in indents[1:]: - yield DEDENT, '', end_pos, '' - yield ENDMARKER, '', end_pos, prefix + yield TokenInfo(DEDENT, '', end_pos, '') + yield TokenInfo(ENDMARKER, '', end_pos, additional_prefix) diff --git a/jedi/parser/tree.py b/jedi/parser/tree.py index 899d5f1c..6013f36a 100644 --- a/jedi/parser/tree.py +++ b/jedi/parser/tree.py @@ -14,8 +14,8 @@ The easiest way to play with this module is to use :class:`parsing.Parser`. :attr:`parsing.Parser.module` holds an instance of :class:`Module`: >>> from jedi._compatibility import u ->>> from jedi.parser import Parser, load_grammar ->>> parser = Parser(load_grammar(), u('import os'), 'example.py') +>>> from jedi.parser import ParserWithRecovery, load_grammar +>>> parser = ParserWithRecovery(load_grammar(), u('import os'), 'example.py') >>> submodule = parser.module >>> submodule @@ -27,16 +27,22 @@ Any subclasses of :class:`Scope`, including :class:`Module` has an attribute [] See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. + +For static analysis purposes there exists a method called +``nodes_to_execute`` on all nodes and leaves. It's documented in the static +anaylsis documentation. """ import os import re from inspect import cleandoc from itertools import chain import textwrap +import abc from jedi._compatibility import (Python3Method, encoding, is_py3, utf8_repr, literal_eval, use_metaclass, unicode) -from jedi import cache +from jedi.parser import token +from jedi.parser.utils import underscore_memoization def is_node(node, *symbol_names): @@ -140,10 +146,131 @@ class Base(object): scope = scope.parent return scope + def get_definition(self): + scope = self + while scope.parent is not None: + parent = scope.parent + if scope.isinstance(Node, Name) and parent.type != 'simple_stmt': + if scope.type == 'testlist_comp': + try: + if isinstance(scope.children[1], CompFor): + return scope.children[1] + except IndexError: + pass + scope = parent + else: + break + return scope + + def assignment_indexes(self): + """ + Returns an array of tuple(int, node) of the indexes that are used in + tuple assignments. + + For example if the name is ``y`` in the following code:: + + x, (y, z) = 2, '' + + would result in ``[(1, xyz_node), (0, yz_node)]``. + """ + indexes = [] + node = self.parent + compare = self + while node is not None: + if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'): + for i, child in enumerate(node.children): + if child == compare: + indexes.insert(0, (int(i / 2), node)) + break + else: + raise LookupError("Couldn't find the assignment.") + elif isinstance(node, (ExprStmt, CompFor)): + break + + compare = node + node = node.parent + return indexes + def is_scope(self): # Default is not being a scope. Just inherit from Scope. return False + @abc.abstractmethod + def nodes_to_execute(self, last_added=False): + raise NotImplementedError() + + def get_next_sibling(self): + """ + The node immediately following the invocant in their parent's children + list. If the invocant does not have a next sibling, it is None + """ + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + try: + return self.parent.children[i + 1] + except IndexError: + return None + + def get_previous_sibling(self): + """ + The node/leaf immediately preceding the invocant in their parent's + children list. If the invocant does not have a previous sibling, it is + None. + """ + # Can't use index(); we need to test by identity + for i, child in enumerate(self.parent.children): + if child is self: + if i == 0: + return None + return self.parent.children[i - 1] + + def get_previous_leaf(self): + """ + Returns the previous leaf in the parser tree. + Raises an IndexError if it's the first element. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + if i == 0: + node = node.parent + if node.parent is None: + raise IndexError('Cannot access the previous element of the first one.') + else: + node = c[i - 1] + break + + while True: + try: + node = node.children[-1] + except AttributeError: # A Leaf doesn't have children. + return node + + def get_next_leaf(self): + """ + Returns the previous leaf in the parser tree. + Raises an IndexError if it's the last element. + """ + node = self + while True: + c = node.parent.children + i = c.index(node) + if i == len(c) - 1: + node = node.parent + if node.parent is None: + raise IndexError('Cannot access the next element of the last one.') + else: + node = c[i + 1] + break + + while True: + try: + node = node.children[0] + except AttributeError: # A Leaf doesn't have children. + return node + class Leaf(Base): __slots__ = ('position_modifier', 'value', 'parent', '_start_pos', 'prefix') @@ -163,6 +290,12 @@ class Leaf(Base): def start_pos(self, value): self._start_pos = value[0] - self.position_modifier.line, value[1] + def get_start_pos_of_prefix(self): + try: + return self.get_previous_leaf().end_pos + except IndexError: + return 1, 0 # It's the first leaf. + @property def end_pos(self): return (self._start_pos[0] + self.position_modifier.line, @@ -172,56 +305,19 @@ class Leaf(Base): self._start_pos = (self._start_pos[0] + line_offset, self._start_pos[1] + column_offset) - def get_previous(self): - """ - Returns the previous leaf in the parser tree. - """ - node = self - while True: - c = node.parent.children - i = c.index(self) - if i == 0: - node = node.parent - if node.parent is None: - raise IndexError('Cannot access the previous element of the first one.') - else: - node = c[i - 1] - break + def first_leaf(self): + return self - while True: - try: - node = node.children[-1] - except AttributeError: # A Leaf doesn't have children. - return node + def get_code(self, normalized=False, include_prefix=True): + if normalized: + return self.value + if include_prefix: + return self.prefix + self.value + else: + return self.value - def get_code(self): - return self.prefix + self.value - - def next_sibling(self): - """ - The node immediately following the invocant in their parent's children - list. If the invocant does not have a next sibling, it is None - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - try: - return self.parent.children[i + 1] - except IndexError: - return None - - def prev_sibling(self): - """ - The node/leaf immediately preceding the invocant in their parent's - children list. If the invocant does not have a previous sibling, it is - None. - """ - # Can't use index(); we need to test by identity - for i, child in enumerate(self.parent.children): - if child is self: - if i == 0: - return None - return self.parent.children[i - 1] + def nodes_to_execute(self, last_added=False): + return [] @utf8_repr def __repr__(self): @@ -247,12 +343,20 @@ class LeafWithNewLines(Leaf): end_pos_col = len(lines[-1]) return end_pos_line, end_pos_col + @utf8_repr + def __repr__(self): + return "<%s: %r>" % (type(self).__name__, self.value) + class Whitespace(LeafWithNewLines): """Contains NEWLINE and ENDMARKER tokens.""" __slots__ = () type = 'whitespace' + @utf8_repr + def __repr__(self): + return "<%s: %s>" % (type(self).__name__, repr(self.value)) + class Name(Leaf): """ @@ -272,63 +376,26 @@ class Name(Leaf): return "<%s: %s@%s,%s>" % (type(self).__name__, self.value, self.start_pos[0], self.start_pos[1]) - def get_definition(self): - scope = self - while scope.parent is not None: - parent = scope.parent - if scope.isinstance(Node, Name) and parent.type != 'simple_stmt': - if scope.type == 'testlist_comp': - try: - if isinstance(scope.children[1], CompFor): - return scope.children[1] - except IndexError: - pass - scope = parent - else: - break - return scope - def is_definition(self): + if self.parent.type in ('power', 'atom_expr'): + # In `self.x = 3` self is not a definition, but x is. + return False + stmt = self.get_definition() if stmt.type in ('funcdef', 'classdef', 'file_input', 'param'): return self == stmt.name elif stmt.type == 'for_stmt': return self.start_pos < stmt.children[2].start_pos elif stmt.type == 'try_stmt': - return self.prev_sibling() == 'as' + return self.get_previous_sibling() == 'as' else: return stmt.type in ('expr_stmt', 'import_name', 'import_from', 'comp_for', 'with_stmt') \ and self in stmt.get_defined_names() - def assignment_indexes(self): - """ - Returns an array of ints of the indexes that are used in tuple - assignments. - - For example if the name is ``y`` in the following code:: - - x, (y, z) = 2, '' - - would result in ``[1, 0]``. - """ - indexes = [] - node = self.parent - compare = self - while node is not None: - if is_node(node, 'testlist_comp', 'testlist_star_expr', 'exprlist'): - for i, child in enumerate(node.children): - if child == compare: - indexes.insert(0, int(i / 2)) - break - else: - raise LookupError("Couldn't find the assignment.") - elif isinstance(node, (ExprStmt, CompFor)): - break - - compare = node - node = node.parent - return indexes + def nodes_to_execute(self, last_added=False): + if last_added is False: + yield self class Literal(LeafWithNewLines): @@ -348,6 +415,16 @@ class String(Literal): __slots__ = () +class Indent(Leaf): + type = 'indent' + __slots__ = () + + +class Dedent(Leaf): + type = 'indent' + __slots__ = () + + class Operator(Leaf): type = 'operator' __slots__ = () @@ -424,12 +501,20 @@ class BaseNode(Base): def start_pos(self): return self.children[0].start_pos + def get_start_pos_of_prefix(self): + return self.children[0].get_start_pos_of_prefix() + @property def end_pos(self): return self.children[-1].end_pos - def get_code(self): - return "".join(c.get_code() for c in self.children) + def get_code(self, normalized=False, include_prefix=True): + # TODO implement normalized (depending on context). + if include_prefix: + return "".join(c.get_code(normalized) for c in self.children) + else: + first = self.children[0].get_code(include_prefix=False) + return first + "".join(c.get_code(normalized) for c in self.children[1:]) @Python3Method def name_for_position(self, position): @@ -443,6 +528,21 @@ class BaseNode(Base): return result return None + def get_leaf_for_position(self, position, include_prefixes=False): + for c in self.children: + if include_prefixes: + start_pos = c.get_start_pos_of_prefix() + else: + start_pos = c.start_pos + + if start_pos <= position <= c.end_pos: + try: + return c.get_leaf_for_position(position, include_prefixes) + except AttributeError: + return c + + return None + @Python3Method def get_statement_for_position(self, pos): for c in self.children: @@ -463,9 +563,54 @@ class BaseNode(Base): except AttributeError: return self.children[0] + def get_next_leaf(self): + """ + Raises an IndexError if it's the last node. (Would be the module) + """ + c = self.parent.children + index = c.index(self) + if index == len(c) - 1: + # TODO WTF? recursion? + return self.get_next_leaf() + else: + return c[index + 1] + + def last_leaf(self): + try: + return self.children[-1].last_leaf() + except AttributeError: + return self.children[-1] + + def get_following_comment_same_line(self): + """ + returns (as string) any comment that appears on the same line, + after the node, including the # + """ + try: + if self.isinstance(ForStmt): + whitespace = self.children[5].first_leaf().prefix + elif self.isinstance(WithStmt): + whitespace = self.children[3].first_leaf().prefix + else: + whitespace = self.last_leaf().get_next_leaf().prefix + except AttributeError: + return None + except ValueError: + # TODO in some particular cases, the tree doesn't seem to be linked + # correctly + return None + if "#" not in whitespace: + return None + comment = whitespace[whitespace.index("#"):] + if "\r" in comment: + comment = comment[:comment.index("\r")] + if "\n" in comment: + comment = comment[:comment.index("\n")] + return comment + @utf8_repr def __repr__(self): - code = self.get_code().replace('\n', ' ') + code = self.get_code().replace('\n', ' ').strip() if not is_py3: code = code.encode(encoding, 'replace') return "<%s: %s@%s,%s>" % \ @@ -476,6 +621,13 @@ class Node(BaseNode): """Concrete implementation for interior nodes.""" __slots__ = ('type',) + _IGNORE_EXECUTE_NODES = set([ + 'suite', 'subscriptlist', 'subscript', 'simple_stmt', 'sliceop', + 'testlist_comp', 'dictorsetmaker', 'trailer', 'decorators', + 'decorated', 'arglist', 'argument', 'exprlist', 'testlist', + 'testlist_safe', 'testlist1' + ]) + def __init__(self, type, children): """ Initializer. @@ -488,10 +640,50 @@ class Node(BaseNode): super(Node, self).__init__(children) self.type = type + def nodes_to_execute(self, last_added=False): + """ + For static analysis. + """ + result = [] + if self.type not in Node._IGNORE_EXECUTE_NODES and not last_added: + result.append(self) + last_added = True + + for child in self.children: + result += child.nodes_to_execute(last_added) + return result + def __repr__(self): return "%s(%s, %r)" % (self.__class__.__name__, self.type, self.children) +class ErrorNode(BaseNode): + """ + TODO doc + """ + __slots__ = () + type = 'error_node' + + def nodes_to_execute(self, last_added=False): + return [] + + +class ErrorLeaf(LeafWithNewLines): + """ + TODO doc + """ + __slots__ = ('original_type') + type = 'error_leaf' + + def __init__(self, position_modifier, original_type, value, start_pos, prefix=''): + super(ErrorLeaf, self).__init__(position_modifier, value, start_pos, prefix) + self.original_type = original_type + + def __repr__(self): + token_type = token.tok_name[self.original_type] + return "<%s: %s, %s)>" % (type(self).__name__, token_type, self.start_pos) + + class IsScopeMeta(type): def __instancecheck__(self, other): return other.is_scope() @@ -587,8 +779,7 @@ class Module(Scope): Depending on the underlying parser this may be a full module or just a part of a module. """ - __slots__ = ('path', 'global_names', 'used_names', '_name', - 'error_statement_stacks') + __slots__ = ('path', 'global_names', 'used_names', '_name') type = 'file_input' def __init__(self, children): @@ -604,7 +795,7 @@ class Module(Scope): self.path = None # Set later. @property - @cache.underscore_memoization + @underscore_memoization def name(self): """ This is used for the goto functions. """ if self.path is None: @@ -637,11 +828,25 @@ class Module(Scope): return True return False + def nodes_to_execute(self, last_added=False): + # Yield itself, class needs to be executed for decorator checks. + result = [] + for child in self.children: + result += child.nodes_to_execute() + return result + class Decorator(BaseNode): type = 'decorator' __slots__ = () + def nodes_to_execute(self, last_added=False): + if self.children[-2] == ')': + node = self.children[-3] + if node != '(': + return node.nodes_to_execute() + return [] + class ClassOrFunc(Scope): __slots__ = () @@ -699,6 +904,34 @@ class Class(ClassOrFunc): sub.get_call_signature(func_name=self.name), docstr) return docstr + def nodes_to_execute(self, last_added=False): + # Yield itself, class needs to be executed for decorator checks. + yield self + # Super arguments. + arglist = self.get_super_arglist() + try: + children = arglist.children + except AttributeError: + if arglist is not None: + for node_to_execute in arglist.nodes_to_execute(): + yield node_to_execute + else: + for argument in children: + if argument.type == 'argument': + # metaclass= or list comprehension or */** + raise NotImplementedError('Metaclasses not implemented') + else: + for node_to_execute in argument.nodes_to_execute(): + yield node_to_execute + + # care for the class suite: + for node in self.children[self.children.index(':'):]: + # This could be easier without the fast parser. But we need to find + # the position of the colon, because everything after it can be a + # part of the class, not just its suite. + for node_to_execute in node.nodes_to_execute(): + yield node_to_execute + def _create_params(parent, argslist_list): """ @@ -726,23 +959,28 @@ def _create_params(parent, argslist_list): if first.type in ('name', 'tfpdef'): if check_python2_nested_param(first): - return [] + return [first] else: return [Param([first], parent)] + elif first == '*': + return [first] else: # argslist is a `typedargslist` or a `varargslist`. children = first.children - params = [] + new_children = [] start = 0 # Start with offset 1, because the end is higher. for end, child in enumerate(children + [None], 1): if child is None or child == ',': - new_children = children[start:end] - if new_children: # Could as well be comma and then end. - if check_python2_nested_param(new_children[0]): - continue - params.append(Param(new_children, parent)) + param_children = children[start:end] + if param_children: # Could as well be comma and then end. + if check_python2_nested_param(param_children[0]): + new_children += param_children + elif param_children[0] == '*' and param_children[1] == ',': + new_children += param_children + else: + new_children.append(Param(param_children, parent)) start = end - return params + return new_children class Function(ClassOrFunc): @@ -769,8 +1007,7 @@ class Function(ClassOrFunc): @property def params(self): - # Contents of parameter lit minus the leading and the trailing . - return self.children[2].children[1:-1] + return [p for p in self.children[2].children if p.type == 'param'] @property def name(self): @@ -786,7 +1023,10 @@ class Function(ClassOrFunc): def annotation(self): try: - return self.children[6] # 6th element: def foo(...) -> bar + if self.children[3] == "->": + return self.children[4] + assert self.children[3] == ":" + return None except IndexError: return None @@ -807,13 +1047,28 @@ class Function(ClassOrFunc): def _get_paramlist_code(self): return self.children[2].get_code() - + @property def doc(self): """ Return a document string including call signature. """ docstr = self.raw_doc return '%s\n\n%s' % (self.get_call_signature(), docstr) + def nodes_to_execute(self, last_added=False): + # Yield itself, functions needs to be executed for decorator checks. + yield self + for param in self.params: + if param.default is not None: + yield param.default + # care for the function suite: + for node in self.children[4:]: + # This could be easier without the fast parser. The fast parser + # allows that the 4th position is empty or that there's even a + # fifth element (another function/class). So just scan everything + # after colon. + for node_to_execute in node.nodes_to_execute(): + yield node_to_execute + class Lambda(Function): """ @@ -842,7 +1097,7 @@ class Lambda(Function): def _get_paramlist_code(self): return '(' + ''.join(param.get_code() for param in self.params).strip() + ')' - + @property def params(self): return self.children[1:-2] @@ -850,10 +1105,22 @@ class Lambda(Function): def is_generator(self): return False + def annotation(self): + # lambda functions do not support annotations + return None + @property def yields(self): return [] + def nodes_to_execute(self, last_added=False): + for param in self.params: + if param.default is not None: + yield param.default + # Care for the lambda test (last child): + for node_to_execute in self.children[-1].nodes_to_execute(): + yield node_to_execute + def __repr__(self): return "<%s@%s>" % (self.__class__.__name__, self.start_pos) @@ -861,6 +1128,11 @@ class Lambda(Function): class Flow(BaseNode): __slots__ = () + def nodes_to_execute(self, last_added=False): + for child in self.children: + for node_to_execute in child.nodes_to_execute(): + yield node_to_execute + class IfStmt(Flow): type = 'if_stmt' @@ -880,9 +1152,20 @@ class IfStmt(Flow): yield self.children[i + 1] def node_in_which_check_node(self, node): + """ + Returns the check node (see function above) that a node is contained + in. However if it the node is in the check node itself and not in the + suite return None. + """ + start_pos = node.start_pos for check_node in reversed(list(self.check_nodes())): - if check_node.start_pos < node.start_pos: - return check_node + if check_node.start_pos < start_pos: + if start_pos < check_node.end_pos: + return None + # In this case the node is within the check_node itself, + # not in the suite + else: + return check_node def node_after_else(self, node): """ @@ -905,6 +1188,21 @@ class ForStmt(Flow): type = 'for_stmt' __slots__ = () + def get_input_node(self): + """ + Returns the input node ``y`` from: ``for x in y:``. + """ + return self.children[3] + + def defines_one_name(self): + """ + Returns True if only one name is returned: ``for x in y``. + Returns False if the for loop is more complicated: ``for x, z in y``. + + :returns: bool + """ + return self.children[1].type == 'name' + class TryStmt(Flow): type = 'try_stmt' @@ -921,6 +1219,16 @@ class TryStmt(Flow): elif node == 'except': yield None + def nodes_to_execute(self, last_added=False): + result = [] + for child in self.children[2::3]: + result += child.nodes_to_execute() + for child in self.children[0::3]: + if child.type == 'except_clause': + # Add the test node and ignore the `as NAME` definition. + result += child.children[1].nodes_to_execute() + return result + class WithStmt(Flow): type = 'with_stmt' @@ -941,6 +1249,16 @@ class WithStmt(Flow): if is_node(node, 'with_item'): return node.children[0] + def nodes_to_execute(self, last_added=False): + result = [] + for child in self.children[1::2]: + if child.type == 'with_item': + # Just ignore the `as EXPR` part - at least for now, because + # most times it's just a name. + child = child.children[0] + result += child.nodes_to_execute() + return result + class Import(BaseNode): __slots__ = () @@ -963,6 +1281,14 @@ class Import(BaseNode): def is_star_import(self): return self.children[-1] == '*' + def nodes_to_execute(self, last_added=False): + """ + `nodes_to_execute` works a bit different for imports, because the names + itself cannot directly get resolved (except on itself). + """ + # TODO couldn't we return the names? Would be nicer. + return [self] + class ImportFrom(Import): type = 'import_from' @@ -1087,17 +1413,33 @@ class ImportName(Import): class KeywordStatement(BaseNode): """ For the following statements: `assert`, `del`, `global`, `nonlocal`, - `raise`, `return`, `yield`, `pass`, `continue`, `break`, `return`, `yield`. + `raise`, `return`, `yield`, `return`, `yield`. + + `pass`, `continue` and `break` are not in there, because they are just + simple keywords and the parser reduces it to a keyword. """ __slots__ = () + @property + def type(self): + """ + Keyword statements start with the keyword and end with `_stmt`. You can + crosscheck this with the Python grammar. + """ + return '%s_stmt' % self.keyword + @property def keyword(self): return self.children[0].value + def nodes_to_execute(self, last_added=False): + result = [] + for child in self.children: + result += child.nodes_to_execute() + return result + class AssertStmt(KeywordStatement): - type = 'assert_stmt' __slots__ = () def assertion(self): @@ -1105,7 +1447,6 @@ class AssertStmt(KeywordStatement): class GlobalStmt(KeywordStatement): - type = 'global_stmt' __slots__ = () def get_defined_names(self): @@ -1114,16 +1455,31 @@ class GlobalStmt(KeywordStatement): def get_global_names(self): return self.children[1::2] + def nodes_to_execute(self, last_added=False): + """ + The global keyword allows to define any name. Even if it doesn't + exist. + """ + return [] + class ReturnStmt(KeywordStatement): - type = 'return_stmt' __slots__ = () class YieldExpr(BaseNode): - type = 'yield_expr' __slots__ = () + @property + def type(self): + return 'yield_expr' + + def nodes_to_execute(self, last_added=False): + if len(self.children) > 1: + return self.children[1].nodes_to_execute() + else: + return [] + def _defined_names(current): """ @@ -1134,9 +1490,9 @@ def _defined_names(current): if is_node(current, 'testlist_star_expr', 'testlist_comp', 'exprlist'): for child in current.children[::2]: names += _defined_names(child) - elif is_node(current, 'atom'): + elif is_node(current, 'atom', 'star_expr'): names += _defined_names(current.children[1]) - elif is_node(current, 'power'): + elif is_node(current, 'power', 'atom_expr'): if current.children[-2] != '**': # Just if there's no operation trailer = current.children[-1] if trailer.children[0] == '.': @@ -1168,6 +1524,14 @@ class ExprStmt(BaseNode, DocstringMixin): except IndexError: return None + def nodes_to_execute(self, last_added=False): + # I think evaluating the statement (and possibly returned arrays), + # should be enough for static analysis. + result = [self] + for child in self.children: + result += child.nodes_to_execute(last_added=True) + return result + class Param(BaseNode): """ @@ -1198,8 +1562,14 @@ class Param(BaseNode): return None def annotation(self): - # Generate from tfpdef. - raise NotImplementedError + tfpdef = self._tfpdef() + if is_node(tfpdef, 'tfpdef'): + assert tfpdef.children[1] == ":" + assert len(tfpdef.children) == 3 + annotation = tfpdef.children[2] + return annotation + else: + return None def _tfpdef(self): """ @@ -1232,6 +1602,16 @@ class CompFor(BaseNode): type = 'comp_for' __slots__ = () + def get_comp_fors(self): + yield self + last = self.children[-1] + while True: + if isinstance(last, CompFor): + yield last + elif not is_node(last, 'comp_if'): + break + last = last.children[-1] + def is_scope(self): return True @@ -1248,3 +1628,16 @@ class CompFor(BaseNode): def get_defined_names(self): return _defined_names(self.children[1]) + + def nodes_to_execute(self, last_added=False): + last = self.children[-1] + if last.type == 'comp_if': + for node in last.children[-1].nodes_to_execute(): + yield node + last = self.children[-2] + elif last.type == 'comp_for': + for node in last.nodes_to_execute(): + yield node + last = self.children[-2] + for node in last.nodes_to_execute(): + yield node diff --git a/jedi/parser/user_context.py b/jedi/parser/user_context.py deleted file mode 100644 index 3cb24a7c..00000000 --- a/jedi/parser/user_context.py +++ /dev/null @@ -1,339 +0,0 @@ -import re -import os -import keyword - -from jedi import cache -from jedi import common -from jedi.parser import tokenize, Parser -from jedi._compatibility import u -from jedi.parser.fast import FastParser -from jedi.parser import tree -from jedi import debug -from jedi.common import PushBackIterator - - -REPLACE_STR = r"[bBuU]?[rR]?" + (r"(?:(')[^\n'\\]*(?:\\.[^\n'\\]*)*(?:'|$)" + - '|' + - r'(")[^\n"\\]*(?:\\.[^\n"\\]*)*(?:"|$))') -REPLACE_STR = re.compile(REPLACE_STR) - - -class UserContext(object): - """ - :param source: The source code of the file. - :param position: The position, the user is currently in. Only important \ - for the main file. - """ - def __init__(self, source, position): - self.source = source - self.position = position - self._line_cache = None - - self._relevant_temp = None - - @cache.underscore_memoization - def get_path_until_cursor(self): - """ Get the path under the cursor. """ - path, self._start_cursor_pos = self._calc_path_until_cursor(self.position) - return path - - def _backwards_line_generator(self, start_pos): - self._line_temp, self._column_temp = start_pos - first_line = self.get_line(start_pos[0])[:self._column_temp] - - self._line_length = self._column_temp - yield first_line[::-1] + '\n' - - while True: - self._line_temp -= 1 - line = self.get_line(self._line_temp) - self._line_length = len(line) - yield line[::-1] + '\n' - - def _get_backwards_tokenizer(self, start_pos, line_gen=None): - if line_gen is None: - line_gen = self._backwards_line_generator(start_pos) - token_gen = tokenize.generate_tokens(lambda: next(line_gen)) - for typ, tok_str, tok_start_pos, prefix in token_gen: - line = self.get_line(self._line_temp) - # Calculate the real start_pos of the token. - if tok_start_pos[0] == 1: - # We are in the first checked line - column = start_pos[1] - tok_start_pos[1] - else: - column = len(line) - tok_start_pos[1] - # Multi-line docstrings must be accounted for. - first_line = common.splitlines(tok_str)[0] - column -= len(first_line) - # Reverse the token again, so that it is in normal order again. - yield typ, tok_str[::-1], (self._line_temp, column), prefix[::-1] - - def _calc_path_until_cursor(self, start_pos): - """ - Something like a reverse tokenizer that tokenizes the reversed strings. - """ - open_brackets = ['(', '[', '{'] - close_brackets = [')', ']', '}'] - - start_cursor = start_pos - gen = PushBackIterator(self._get_backwards_tokenizer(start_pos)) - string = u('') - level = 0 - force_point = False - last_type = None - is_first = True - for tok_type, tok_str, tok_start_pos, prefix in gen: - if is_first: - if prefix: # whitespace is not a path - return u(''), start_cursor - is_first = False - - if last_type == tok_type == tokenize.NAME: - string = ' ' + string - - if level: - if tok_str in close_brackets: - level += 1 - elif tok_str in open_brackets: - level -= 1 - elif tok_str == '.': - force_point = False - elif force_point: - # Reversed tokenizing, therefore a number is recognized as a - # floating point number. - # The same is true for string prefixes -> represented as a - # combination of string and name. - if tok_type == tokenize.NUMBER and tok_str[-1] == '.' \ - or tok_type == tokenize.NAME and last_type == tokenize.STRING \ - and tok_str.lower() in ('b', 'u', 'r', 'br', 'ur'): - force_point = False - else: - break - elif tok_str in close_brackets: - level += 1 - elif tok_type in [tokenize.NAME, tokenize.STRING]: - if keyword.iskeyword(tok_str) and string: - # If there's already something in the string, a keyword - # never adds any meaning to the current statement. - break - force_point = True - elif tok_type == tokenize.NUMBER: - pass - else: - if tok_str == '-': - next_tok = next(gen) - if next_tok[1] == 'e': - gen.push_back(next_tok) - else: - break - else: - break - - start_cursor = tok_start_pos - string = tok_str + prefix + string - last_type = tok_type - - # Don't need whitespace around a statement. - return string.strip(), start_cursor - - def get_path_under_cursor(self): - """ - Return the path under the cursor. If there is a rest of the path left, - it will be added to the stuff before it. - """ - return self.get_path_until_cursor() + self.get_path_after_cursor() - - def get_path_after_cursor(self): - line = self.get_line(self.position[0]) - return re.search("[\w\d]*", line[self.position[1]:]).group(0) - - def get_operator_under_cursor(self): - line = self.get_line(self.position[0]) - after = re.match("[^\w\s]+", line[self.position[1]:]) - before = re.match("[^\w\s]+", line[:self.position[1]][::-1]) - return (before.group(0) if before is not None else '') \ - + (after.group(0) if after is not None else '') - - def call_signature(self): - """ - :return: Tuple of string of the call and the index of the cursor. - """ - def get_line(pos): - def simplify_str(match): - """ - To avoid having strings without end marks (error tokens) and - strings that just screw up all the call signatures, just - simplify everything. - """ - mark = match.group(1) or match.group(2) - return mark + ' ' * (len(match.group(0)) - 2) + mark - - line_gen = self._backwards_line_generator(pos) - for line in line_gen: - # We have to switch the already backwards lines twice, because - # we scan them from start. - line = line[::-1] - modified = re.sub(REPLACE_STR, simplify_str, line) - yield modified[::-1] - - index = 0 - level = 0 - next_must_be_name = False - next_is_key = False - key_name = None - generator = self._get_backwards_tokenizer(self.position, get_line(self.position)) - for tok_type, tok_str, start_pos, prefix in generator: - if tok_str in tokenize.ALWAYS_BREAK_TOKENS: - break - elif next_must_be_name: - if tok_type == tokenize.NUMBER: - # If there's a number at the end of the string, it will be - # tokenized as a number. So add it to the name. - tok_type, t, _, _ = next(generator) - if tok_type == tokenize.NAME: - end_pos = start_pos[0], start_pos[1] + len(tok_str) - call, start_pos = self._calc_path_until_cursor(start_pos=end_pos) - return call, index, key_name, start_pos - index = 0 - next_must_be_name = False - elif next_is_key: - if tok_type == tokenize.NAME: - key_name = tok_str - next_is_key = False - - if tok_str == '(': - level += 1 - if level == 1: - next_must_be_name = True - level = 0 - elif tok_str == ')': - level -= 1 - elif tok_str == ',': - index += 1 - elif tok_str == '=': - next_is_key = True - return None, 0, None, (0, 0) - - def get_context(self, yield_positions=False): - self.get_path_until_cursor() # In case _start_cursor_pos is undefined. - pos = self._start_cursor_pos - while True: - # remove non important white space - line = self.get_line(pos[0]) - while True: - if pos[1] == 0: - line = self.get_line(pos[0] - 1) - if line and line[-1] == '\\': - pos = pos[0] - 1, len(line) - 1 - continue - else: - break - - if line[pos[1] - 1].isspace(): - pos = pos[0], pos[1] - 1 - else: - break - - try: - result, pos = self._calc_path_until_cursor(start_pos=pos) - if yield_positions: - yield pos - else: - yield result - except StopIteration: - if yield_positions: - yield None - else: - yield '' - - def get_line(self, line_nr): - if not self._line_cache: - self._line_cache = common.splitlines(self.source) - - if line_nr == 0: - # This is a fix for the zeroth line. We need a newline there, for - # the backwards parser. - return u('') - if line_nr < 0: - raise StopIteration() - try: - return self._line_cache[line_nr - 1] - except IndexError: - raise StopIteration() - - def get_position_line(self): - return self.get_line(self.position[0])[:self.position[1]] - - -class UserContextParser(object): - def __init__(self, grammar, source, path, position, user_context, - parser_done_callback, use_fast_parser=True): - self._grammar = grammar - self._source = source - self._path = path and os.path.abspath(path) - self._position = position - self._user_context = user_context - self._use_fast_parser = use_fast_parser - self._parser_done_callback = parser_done_callback - - @cache.underscore_memoization - def _parser(self): - cache.invalidate_star_import_cache(self._path) - if self._use_fast_parser: - parser = FastParser(self._grammar, self._source, self._path) - # Don't pickle that module, because the main module is changing quickly - cache.save_parser(self._path, parser, pickling=False) - else: - parser = Parser(self._grammar, self._source, self._path) - self._parser_done_callback(parser) - return parser - - @cache.underscore_memoization - def user_stmt(self): - module = self.module() - debug.speed('parsed') - return module.get_statement_for_position(self._position) - - @cache.underscore_memoization - def user_stmt_with_whitespace(self): - """ - Returns the statement under the cursor even if the statement lies - before the cursor. - """ - user_stmt = self.user_stmt() - - if not user_stmt: - # for statements like `from x import ` (cursor not in statement) - # or `abs( ` where the cursor is out in the whitespace. - if self._user_context.get_path_under_cursor(): - # We really should have a user_stmt, but the parser couldn't - # process it - probably a Syntax Error (or in a comment). - debug.warning('No statement under the cursor.') - return - pos = next(self._user_context.get_context(yield_positions=True)) - user_stmt = self.module().get_statement_for_position(pos) - return user_stmt - - @cache.underscore_memoization - def user_scope(self): - """ - Returns the scope in which the user resides. This includes flows. - """ - user_stmt = self.user_stmt() - if user_stmt is None: - def scan(scope): - for s in scope.children: - if s.start_pos <= self._position <= s.end_pos: - if isinstance(s, (tree.Scope, tree.Flow)): - if isinstance(s, tree.Flow): - return s - return scan(s) or s - elif s.type in ('suite', 'decorated'): - return scan(s) - - return scan(self.module()) or self.module() - else: - return user_stmt.get_parent_scope(include_flows=True) - - def module(self): - return self._parser().module diff --git a/jedi/parser/utils.py b/jedi/parser/utils.py new file mode 100644 index 00000000..fadb0b87 --- /dev/null +++ b/jedi/parser/utils.py @@ -0,0 +1,198 @@ +import inspect +import time +import os +import sys +import json +import hashlib +import gc +import shutil +import pickle + +from jedi import settings +from jedi import debug + + +def underscore_memoization(func): + """ + Decorator for methods:: + + class A(object): + def x(self): + if self._x: + self._x = 10 + return self._x + + Becomes:: + + class A(object): + @underscore_memoization + def x(self): + return 10 + + A now has an attribute ``_x`` written by this decorator. + """ + name = '_' + func.__name__ + + def wrapper(self): + try: + return getattr(self, name) + except AttributeError: + result = func(self) + if inspect.isgenerator(result): + result = list(result) + setattr(self, name, result) + return result + + return wrapper + + +# for fast_parser, should not be deleted +parser_cache = {} + + +class ParserCacheItem(object): + def __init__(self, parser, change_time=None): + self.parser = parser + if change_time is None: + change_time = time.time() + self.change_time = change_time + + +def load_parser(path): + """ + Returns the module or None, if it fails. + """ + p_time = os.path.getmtime(path) if path else None + try: + parser_cache_item = parser_cache[path] + if not path or p_time <= parser_cache_item.change_time: + return parser_cache_item.parser + except KeyError: + if settings.use_filesystem_cache: + return ParserPickling.load_parser(path, p_time) + + +def save_parser(path, parser, pickling=True): + try: + p_time = None if path is None else os.path.getmtime(path) + except OSError: + p_time = None + pickling = False + + item = ParserCacheItem(parser, p_time) + parser_cache[path] = item + if settings.use_filesystem_cache and pickling: + ParserPickling.save_parser(path, item) + + +class ParserPickling(object): + + version = 25 + """ + Version number (integer) for file system cache. + + Increment this number when there are any incompatible changes in + parser representation classes. For example, the following changes + are regarded as incompatible. + + - Class name is changed. + - Class is moved to another module. + - Defined slot of the class is changed. + """ + + def __init__(self): + self.__index = None + self.py_tag = 'cpython-%s%s' % sys.version_info[:2] + """ + Short name for distinguish Python implementations and versions. + + It's like `sys.implementation.cache_tag` but for Python < 3.3 + we generate something similar. See: + http://docs.python.org/3/library/sys.html#sys.implementation + + .. todo:: Detect interpreter (e.g., PyPy). + """ + + def load_parser(self, path, original_changed_time): + try: + pickle_changed_time = self._index[path] + except KeyError: + return None + if original_changed_time is not None \ + and pickle_changed_time < original_changed_time: + # the pickle file is outdated + return None + + with open(self._get_hashed_path(path), 'rb') as f: + try: + gc.disable() + parser_cache_item = pickle.load(f) + finally: + gc.enable() + + debug.dbg('pickle loaded: %s', path) + parser_cache[path] = parser_cache_item + return parser_cache_item.parser + + def save_parser(self, path, parser_cache_item): + self.__index = None + try: + files = self._index + except KeyError: + files = {} + self._index = files + + with open(self._get_hashed_path(path), 'wb') as f: + pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL) + files[path] = parser_cache_item.change_time + + self._flush_index() + + @property + def _index(self): + if self.__index is None: + try: + with open(self._get_path('index.json')) as f: + data = json.load(f) + except (IOError, ValueError): + self.__index = {} + else: + # 0 means version is not defined (= always delete cache): + if data.get('version', 0) != self.version: + self.clear_cache() + self.__index = {} + else: + self.__index = data['index'] + return self.__index + + def _remove_old_modules(self): + # TODO use + change = False + if change: + self._flush_index(self) + self._index # reload index + + def _flush_index(self): + data = {'version': self.version, 'index': self._index} + with open(self._get_path('index.json'), 'w') as f: + json.dump(data, f) + self.__index = None + + def clear_cache(self): + shutil.rmtree(self._cache_directory()) + + def _get_hashed_path(self, path): + return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest()) + + def _get_path(self, file): + dir = self._cache_directory() + if not os.path.exists(dir): + os.makedirs(dir) + return os.path.join(dir, file) + + def _cache_directory(self): + return os.path.join(settings.cache_directory, self.py_tag) + + +# is a singleton +ParserPickling = ParserPickling() diff --git a/jedi/settings.py b/jedi/settings.py index fb0b38f5..13dcfd45 100644 --- a/jedi/settings.py +++ b/jedi/settings.py @@ -63,6 +63,7 @@ definitely worse in some cases. But a completion should also be fast. .. autodata:: max_function_recursion_level .. autodata:: max_executions_without_builtins .. autodata:: max_executions +.. autodata:: max_dynamic_params_depth .. autodata:: scale_call_signatures @@ -86,13 +87,6 @@ case_insensitive_completion = True The completion is by default case insensitive. """ -add_dot_after_module = False -""" -Adds a dot after a module, because a module that is not accessed this way is -definitely not the normal case. However, in VIM this doesn't work, that's why -it isn't used at the moment. -""" - add_bracket_after_function = False """ Adds an opening bracket after a function, because that's normal behaviour. diff --git a/jedi/utils.py b/jedi/utils.py index 6b48ef49..848d8571 100644 --- a/jedi/utils.py +++ b/jedi/utils.py @@ -10,8 +10,8 @@ import os import sys from jedi import Interpreter -from jedi.api.helpers import completion_parts -from jedi.parser.user_context import UserContext +from jedi.api.helpers import get_on_completion_name +from jedi import common def setup_readline(namespace_module=__main__): @@ -72,9 +72,10 @@ def setup_readline(namespace_module=__main__): try: interpreter = Interpreter(text, [namespace_module.__dict__]) - path = UserContext(text, (1, len(text))).get_path_until_cursor() - path, dot, like = completion_parts(path) - before = text[:len(text) - len(like)] + lines = common.splitlines(text) + position = (len(lines), len(lines[-1])) + name = get_on_completion_name(lines, position) + before = text[:len(text) - len(name)] completions = interpreter.completions() finally: sys.path.pop(0) @@ -88,7 +89,7 @@ def setup_readline(namespace_module=__main__): try: import readline except ImportError: - print("Module readline not available.") + print("Jedi: Module readline not available.") else: readline.set_completer(JediRL().complete) readline.parse_and_bind("tab: complete") diff --git a/setup.py b/setup.py index 85dee723..89ff880b 100755 --- a/setup.py +++ b/setup.py @@ -40,9 +40,9 @@ setup(name='jedi', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Text Editors :: Integrated Development Environments (IDE)', 'Topic :: Utilities', diff --git a/test/completion/arrays.py b/test/completion/arrays.py index 1dc896e1..938b9fdf 100644 --- a/test/completion/arrays.py +++ b/test/completion/arrays.py @@ -39,13 +39,16 @@ b[8:] #? list() b[int():] +#? list() +b[:] + class _StrangeSlice(): - def __getitem__(self, slice): - return slice + def __getitem__(self, sliced): + return sliced # Should not result in an error, just because the slice itself is returned. -#? [] +#? slice() _StrangeSlice()[1:2] @@ -125,6 +128,9 @@ f # ----------------- # unnessecary braces # ----------------- +a = (1) +#? int() +a #? int() (1) #? int() @@ -204,9 +210,22 @@ dic2['asdf'] dic2[r'asdf'] #? int() dic2[r'asdf'] +#? int() +dic2[r'as' 'd' u'f'] #? int() str() dic2['just_something'] +# unpacking +a, b = dic2 +#? str() +a +a, b = {1: 'x', 2.0: 1j} +#? int() float() +a +#? int() float() +b + + def f(): """ github #83 """ r = {} @@ -232,6 +251,11 @@ dic = {str(key): ''} #? str() dic[''] + +for x in {1: 3.0, '': 1j}: + #? int() str() + x + # ----------------- # with variable as index # ----------------- @@ -365,3 +389,45 @@ recursion1([1,2])[0] for x in [1] + ['']: #? int() str() x + +# ----------------- +# For loops with attribute assignment. +# ----------------- +def test_func(): + x = 'asdf' + for x.something in [6,7,8]: + pass + #? str() + x + + for x.something, b in [[6, 6.0]]: + pass + #? str() + x + + +# ----------------- +# PEP 3132 Extended Iterable Unpacking (star unpacking) +# ----------------- + +a, *b, c = [1, 'b', list, dict] +#? int() +a +#? str() +b +#? list +c + +# Not valid syntax +a, *b, *c = [1, 'd', list] +#? int() +a +#? str() +b +#? list +c + +lc = [x for a, *x in [(1, '', 1.0)]] + +#? +lc[0][0] diff --git a/test/completion/basic.py b/test/completion/basic.py index 8337b8dd..9b98fbaf 100644 --- a/test/completion/basic.py +++ b/test/completion/basic.py @@ -165,6 +165,17 @@ def global_define(): #? int() global_var_in_func + +def funct1(): + # From issue #610 + global global_dict_var + global_dict_var = dict() +def funct2(): + global global_dict_var + #? dict() + global_dict_var + + # ----------------- # within docstrs # ----------------- @@ -215,6 +226,9 @@ if 1: #? str() xyz +#? +Âą. + # ----------------- # exceptions # ----------------- diff --git a/test/completion/classes.py b/test/completion/classes.py index 1481c6a9..35b0619c 100644 --- a/test/completion/classes.py +++ b/test/completion/classes.py @@ -83,6 +83,9 @@ TestClass.var_local. #? int() TestClass().ret(1) +# Should not return int(), because we want the type before `.ret(1)`. +#? 11 TestClass() +TestClass().ret(1) #? int() inst.ret(1) @@ -131,6 +134,8 @@ A().addition A().addition = None #? 8 int() A(1).addition = None +#? 1 A +A(1).addition = None a = A() #? 8 int() a.addition = None @@ -238,7 +243,10 @@ class V: V(1).b() #? int() V(1).c() -#? [] +#? +V(1).d() +# Only keywords should be possible to complete. +#? ['is', 'in', 'not', 'and', 'or', 'if'] V(1).d() diff --git a/test/completion/comprehensions.py b/test/completion/comprehensions.py index 98a15bcb..78e4f4b3 100644 --- a/test/completion/comprehensions.py +++ b/test/completion/comprehensions.py @@ -30,8 +30,12 @@ a[0] arr = [1,''] a = [a for a in arr] -#? int() str() +#? int() a[0] +#? str() +a[1] +#? int() str() +a[2] a = [a if 1.0 else '' for a in [1] if [1.0]] #? int() str() @@ -44,12 +48,9 @@ left, right = [x for x in (left, right)] left # with a dict literal -#? str() +#? int() [a for a in {1:'x'}][0] -##? str() -{a-1:b for a,b in {1:'a', 3:1.0}.items()}[0] - # list comprehensions should also work in combination with functions def listen(arg): for x in arg: @@ -57,9 +58,12 @@ def listen(arg): x listen(['' for x in [1]]) -#? str +#? ([str for x in []])[0] +# with a set literal +#? int() +[a for a in {1, 2, 3}][0] # ----------------- # nested list comprehensions @@ -93,6 +97,8 @@ left, right = (i for i in (1, '')) #? int() left +#? str() +right gen = (i for i in (1,)) @@ -110,9 +116,52 @@ next(gen) # issues with different formats left, right = (i for i in - ('1', '2')) + ('1', 2)) #? str() left +#? int() +right + +# ----------------- +# dict comprehensions +# ----------------- + +#? int() +list({a - 1: 3 for a in [1]})[0] + +d = {a - 1: b for a, b in {1: 'a', 3: 1.0}.items()} +#? int() +list(d)[0] +#? str() float() +d.values()[0] +#? str() +d[0] +#? float() str() +d[1] +#? float() +d[2] + +# ----------------- +# set comprehensions +# ----------------- + +#? set() +{a - 1 for a in [1]} + +#? set() +{a for a in range(10)} + +#? int() +[x for x in {a for a in range(10)}][0] + +#? int() +{a for a in range(10)}.pop() +#? float() str() +{b for a in [[3.0], ['']] for b in a}.pop() + +#? int() +next(iter({a for a in range(10)})) + # ----------------- # name resolution in comprehensions. @@ -123,3 +172,5 @@ def x(): #? 22 [a for a in h if hio] if hio: pass + + diff --git a/test/completion/decorators.py b/test/completion/decorators.py index 05ba5775..97460a4a 100644 --- a/test/completion/decorators.py +++ b/test/completion/decorators.py @@ -38,7 +38,7 @@ exe[1] #? set exe[2] #? [] -exe[3][0] +exe[3][0]. #? str() exe[4]['d'] @@ -70,7 +70,7 @@ exe[1] #? set exe[2] #? [] -exe[3][0] +exe[3][0]. #? str() exe[4]['d'] @@ -78,6 +78,9 @@ exe[4]['d'] # ----------------- # Decorator is a class # ----------------- +def same_func(func): + return func + class Decorator(object): def __init__(self, func): self.func = func @@ -94,10 +97,15 @@ nothing("")[0] #? str() nothing("")[1] + +@same_func @Decorator def nothing(a,b,c): return a,b,c +#? int() +nothing("")[0] + class MethodDecoratorAsClass(): class_var = 3 @Decorator diff --git a/test/completion/definition.py b/test/completion/definition.py index 19934b87..f896984e 100644 --- a/test/completion/definition.py +++ b/test/completion/definition.py @@ -6,6 +6,9 @@ Fallback to callee definition when definition not found. """Parenthesis closed at next line.""" +# Ignore these definitions for a little while, not sure if we really want them. +# python <= 2.5 + #? isinstance isinstance( ) diff --git a/test/completion/descriptors.py b/test/completion/descriptors.py index 86e3a3d0..4741778b 100644 --- a/test/completion/descriptors.py +++ b/test/completion/descriptors.py @@ -62,14 +62,14 @@ class B(): p = property(t) #? [] -B().r() +B().r(). #? int() B().r #? str() B().p #? [] -B().p() +B().p(). class PropClass(): def __init__(self, a): diff --git a/test/completion/flow_analysis.py b/test/completion/flow_analysis.py index 933badf5..2840d9ee 100644 --- a/test/completion/flow_analysis.py +++ b/test/completion/flow_analysis.py @@ -218,3 +218,40 @@ else: a = '' #? int() a + + +# ----------------- +# Recursion issues +# ----------------- + +def possible_recursion_error(filename): + if filename == 'a': + return filename + # It seems like without the brackets there wouldn't be a RecursionError. + elif type(filename) == str: + return filename + + +if NOT_DEFINED: + s = str() +else: + s = str() +#? str() +possible_recursion_error(s) + + +# ----------------- +# In combination with imports +# ----------------- + +from import_tree import flow_import + +if 1 == flow_import.env: + a = 1 +elif 2 == flow_import.env: + a = '' +elif 3 == flow_import.env: + a = 1.0 + +#? int() str() +a diff --git a/test/completion/functions.py b/test/completion/functions.py index 7adebb2a..c1a40e56 100644 --- a/test/completion/functions.py +++ b/test/completion/functions.py @@ -178,6 +178,23 @@ nested_default(a=1.0)[1] #? str() nested_default(a=1.0, b='')[1] +# Defaults should only work if they are defined before - not after. +def default_function(a=default): + #? + return a + +#? +default_function() + +default = int() + +def default_function(a=default): + #? int() + return a + +#? int() +default_function() + # ----------------- # closures @@ -185,8 +202,9 @@ nested_default(a=1.0, b='')[1] def a(): l = 3 def func_b(): - #? str() l = '' + #? str() + l #? ['func_b'] func_b #? int() @@ -327,6 +345,15 @@ exe[3] #? set exe[3]['c'] + +def kwargs_iteration(**kwargs): + return kwargs + +for x in kwargs_iteration(d=3): + #? float() + {'d': 1.0, 'c': '1'}[x] + + # ----------------- # nested *args # ----------------- @@ -342,12 +369,12 @@ def nested_args2(*args, **kwargs): #? int() nested_args('', 1, 1.0, list) #? [] -nested_args('') +nested_args(''). #? int() nested_args2('', 1, 1.0) #? [] -nested_args2('') +nested_args2(''). # ----------------- # nested **kwargs @@ -371,9 +398,9 @@ nested_kw(a=3.0, b=1) #? int() nested_kw(b=1, a=r"") #? [] -nested_kw(1, '') +nested_kw(1, ''). #? [] -nested_kw(a='') +nested_kw(a=''). #? int() nested_kw2(b=1) @@ -382,9 +409,9 @@ nested_kw2(b=1, c=1.0) #? int() nested_kw2(c=1.0, b=1) #? [] -nested_kw2('') +nested_kw2(''). #? [] -nested_kw2(a='') +nested_kw2(a=''). #? [] nested_kw2('', b=1). @@ -404,14 +431,14 @@ nested_both('', b=1, c=1.0, list) nested_both('', c=1.0, b=1, list) #? [] -nested_both('') +nested_both(''). #? int() nested_both2('', b=1, c=1.0) #? int() nested_both2('', c=1.0, b=1) #? [] -nested_both2('') +nested_both2(''). # ----------------- # nested *args/**kwargs with a default arg @@ -438,7 +465,7 @@ nested_def2('', b=1, c=1.0)[1] #? int() nested_def2('', c=1.0, b=1)[1] #? [] -nested_def2('')[1] +nested_def2('')[1]. # ----------------- # magic methods diff --git a/test/completion/generators.py b/test/completion/generators.py index 96aa2b66..d8eba2dd 100644 --- a/test/completion/generators.py +++ b/test/completion/generators.py @@ -22,7 +22,7 @@ def gen_ret(value): next(gen_ret(1)) #? [] -next(gen_ret()) +next(gen_ret()). # generators evaluate to true if cast by bool. a = '' @@ -42,7 +42,7 @@ def get(param): yield "" #? [] -get()[0] +get()[0]. # ----------------- # __iter__ @@ -131,6 +131,18 @@ def simple(): yield "" a, b = simple() +#? int() str() +a +# For now this is ok. +#? +b + + +def simple2(): + yield 1 + yield "" + +a, b = simple2() #? int() a #? str() @@ -163,3 +175,25 @@ gen().send() #? gen()() + +# ----------------- +# yield from +# ----------------- + +# python >= 3.3 + +def yield_from(): + yield from iter([1]) + +#? int() +next(yield_from()) + +def yield_from_multiple(): + yield from iter([1]) + yield str() + +x, y = yield_from_multiple() +#? int() +x +#? str() +y diff --git a/test/completion/goto.py b/test/completion/goto.py index 84bcc2c8..4e178af1 100644 --- a/test/completion/goto.py +++ b/test/completion/goto.py @@ -30,6 +30,12 @@ b = math #! ['b = math'] b +#! 18 ['foo = 10'] +foo = 10;print(foo) + +# ----------------- +# classes +# ----------------- class C(object): def b(self): #! ['b = math'] diff --git a/test/completion/import_tree/flow_import.py b/test/completion/import_tree/flow_import.py new file mode 100644 index 00000000..a0a779ec --- /dev/null +++ b/test/completion/import_tree/flow_import.py @@ -0,0 +1,4 @@ +if name: + env = 1 +else: + env = 2 diff --git a/test/completion/imports.py b/test/completion/imports.py index 2e5509d1..246b91e5 100644 --- a/test/completion/imports.py +++ b/test/completion/imports.py @@ -49,6 +49,8 @@ def scope_nested(): #? float() import_tree.pkg.mod1.a + #? ['a', '__name__', '__package__', '__file__', '__doc__'] + a = import_tree.pkg.mod1. import import_tree.random #? set @@ -75,6 +77,7 @@ def scope_from_import_variable(): without the use of ``sys.modules`` modifications (e.g. ``os.path`` see also github issue #213 for clarification. """ + a = 3 #? from import_tree.mod2.fake import a #? diff --git a/test/completion/invalid.py b/test/completion/invalid.py index 97b3b7a2..7c047e66 100644 --- a/test/completion/invalid.py +++ b/test/completion/invalid.py @@ -31,7 +31,8 @@ def wrong_indents(): asdf = 3 asdf asdf( - #? int() + # TODO this seems to be wrong now? + ##? int() asdf def openbrace(): asdf = 3 @@ -101,7 +102,7 @@ if isi try: except TypeError: #? str() - "" + str() def break(): pass # wrong ternary expression @@ -175,16 +176,16 @@ import datetime as call = '' invalid = .call -#? +#? invalid invalid = call?.call -#? +#? str() invalid # comma invalid = ,call -#? +#? str() invalid diff --git a/test/completion/isinstance.py b/test/completion/isinstance.py index e44312f5..71011502 100644 --- a/test/completion/isinstance.py +++ b/test/completion/isinstance.py @@ -29,6 +29,14 @@ if 2: #? str() ass +# ----------------- +# invalid arguments +# ----------------- + +if isinstance(wrong, str()): + #? + wrong + # ----------------- # in functions # ----------------- @@ -55,6 +63,17 @@ a fooooo2('') +def isinstance_func(arr): + for value in arr: + if isinstance(value, dict): + # Shouldn't fail, even with the dot. + #? 17 dict() + value. + elif isinstance(value, int): + x = value + #? int() + x + # ----------------- # Names with multiple indices. # ----------------- @@ -72,3 +91,10 @@ class Test(): self.testing #? Test() self + +# ----------------- +# Syntax +# ----------------- + +#? +isinstance(1, int()) diff --git a/test/completion/keywords.py b/test/completion/keywords.py index 851140b1..9631e8d6 100644 --- a/test/completion/keywords.py +++ b/test/completion/keywords.py @@ -2,5 +2,58 @@ #? ['raise'] raise -#? ['except', 'Exception'] +#? ['Exception'] except + +#? [] +b + continu + +#? [] +b + continue + +#? ['continue'] +b; continue + +#? ['continue'] +b; continu + +#? [] +c + brea + +#? [] +a + break + +#? ['break'] +b; break + +# ----------------- +# Keywords should not appear everywhere. +# ----------------- + +#? [] +with open() as f +#? [] +def i +#? [] +class i + +#? [] +continue i + +# More syntax details, e.g. while only after newline, but not after semicolon, +# continue also after semicolon +#? ['while'] +while +#? [] +x while +#? [] +x; while +#? ['continue'] +x; continue + +#? [] +and +#? ['and'] +x and +#? [] +x * and diff --git a/test/completion/named_param.py b/test/completion/named_param.py index 571bb076..2dd147e9 100644 --- a/test/completion/named_param.py +++ b/test/completion/named_param.py @@ -20,3 +20,12 @@ a(some_args) #? 13 [] a(some_kwargs) + +def multiple(foo, bar): + pass + +#? 17 ['bar'] +multiple(foo, bar) + +#? ['bar'] +multiple(foo, bar diff --git a/test/completion/on_import.py b/test/completion/on_import.py index ee826f15..dfaad45a 100644 --- a/test/completion/on_import.py +++ b/test/completion/on_import.py @@ -63,9 +63,13 @@ import datetime.date #? 21 ['import'] from import_tree.pkg import pkg +#? 49 ['a', '__name__', '__doc__', '__file__', '__package__'] +from import_tree.pkg.mod1 import not_existant, # whitespace before +#? ['a', '__name__', '__doc__', '__file__', '__package__'] +from import_tree.pkg.mod1 import not_existant, #? 22 ['mod1'] from import_tree.pkg. import mod1 -#? 17 ['mod1', 'mod2', 'random', 'pkg', 'rename1', 'rename2', 'recurse_class1', 'recurse_class2', 'invisible_pkg'] +#? 17 ['mod1', 'mod2', 'random', 'pkg', 'rename1', 'rename2', 'recurse_class1', 'recurse_class2', 'invisible_pkg', 'flow_import'] from import_tree. import pkg #? 18 ['pkg'] diff --git a/test/completion/ordering.py b/test/completion/ordering.py index 2c7e7fe5..61eb1928 100644 --- a/test/completion/ordering.py +++ b/test/completion/ordering.py @@ -87,7 +87,7 @@ from os import path # should not return a function, because `a` is a function above def f(b, a): return a #? [] -f(b=3) +f(b=3). # ----------------- # closure diff --git a/test/completion/parser.py b/test/completion/parser.py index 2f6fbb3f..68793f4f 100644 --- a/test/completion/parser.py +++ b/test/completion/parser.py @@ -1,5 +1,5 @@ """ -Issues with the parser not the completion engine should be here. +Issues with the parser and not the type inference should be part of this file. """ class IndentIssues(): @@ -32,5 +32,12 @@ Just because there's a def keyword, doesn't mean it should not be able to complete to definition. """ definition = 0 -#? ['definition', 'def'] +#? ['definition'] str(def + + +# It might be hard to determine the context +class Foo(object): + @property + #? ['str'] + def bar(str diff --git a/test/completion/pep0484.py b/test/completion/pep0484.py new file mode 100644 index 00000000..b944cd1d --- /dev/null +++ b/test/completion/pep0484.py @@ -0,0 +1,160 @@ +""" Pep-0484 type hinting """ + +# python >= 3.2 + + +class A(): + pass + + +def function_parameters(a: A, b, c: str, d: int, e: str, f: str, g: int=4): + """ + :param e: if docstring and annotation agree, only one should be returned + :type e: str + :param f: if docstring and annotation disagree, both should be returned + :type f: int + """ + #? A() + a + #? + b + #? str() + c + #? int() + d + #? str() + e + #? int() str() + f + # int() + g + + +def return_unspecified(): + pass + +#? +return_unspecified() + + +def return_none() -> None: + """ + Return type None means the same as no return type as far as jedi + is concerned + """ + pass + +#? +return_none() + + +def return_str() -> str: + pass + +#? str() +return_str() + + +def return_custom_class() -> A: + pass + +#? A() +return_custom_class() + + +def return_annotation_and_docstring() -> str: + """ + :rtype: int + """ + pass + +#? str() int() +return_annotation_and_docstring() + + +def return_annotation_and_docstring_different() -> str: + """ + :rtype: str + """ + pass + +#? str() +return_annotation_and_docstring_different() + + +def annotation_forward_reference(b: "B") -> "B": + #? B() + b + +#? ["test_element"] +annotation_forward_reference(1).t + +class B: + test_element = 1 + pass + +#? B() +annotation_forward_reference(1) + + +class SelfReference: + test_element = 1 + def test_method(self, x: "SelfReference") -> "SelfReference": + #? SelfReference() + x + #? ["test_element", "test_method"] + self.t + #? ["test_element", "test_method"] + x.t + #? ["test_element", "test_method"] + self.test_method(1).t + +#? SelfReference() +SelfReference().test_method() + +def function_with_non_pep_0484_annotation( + x: "I can put anything here", + xx: "", + yy: "\r\n\0;+*&^564835(---^&*34", + y: 3 + 3, + zz: float) -> int("42"): + # infers int from function call + #? int() + x + # infers int from function call + #? int() + xx + # infers int from function call + #? int() + yy + # infers str from function call + #? str() + y + #? float() + zz +#? +function_with_non_pep_0484_annotation(1, 2, 3, "force string") + +def function_forward_reference_dynamic( + x: return_str_type(), + y: "return_str_type()") -> None: + #? + x + #? str() + y + +def return_str_type(): + return str + + +X = str +def function_with_assined_class_in_reference(x: X, y: "Y"): + #? str() + x + #? int() + y +Y = int + +def just_because_we_can(x: "flo" + "at"): + #? float() + x diff --git a/test/completion/pep0484_comments.py b/test/completion/pep0484_comments.py new file mode 100644 index 00000000..7d5f7c2e --- /dev/null +++ b/test/completion/pep0484_comments.py @@ -0,0 +1,109 @@ +a = 3 # type: str +#? str() +a + +b = 3 # type: str but I write more +#? int() +b + +c = 3 # type: str # I comment more +#? str() +c + +d = "It should not read comments from the next line" +# type: int +#? str() +d + +# type: int +e = "It should not read comments from the previous line" +#? str() +e + +class BB: pass + +def test(a, b): + a = a # type: BB + c = a # type: str + d = a + # type: str + e = a # type: str # Should ignore long whitespace + + #? BB() + a + #? str() + c + #? BB() + d + #? str() + e + +a,b = 1, 2 # type: str, float +#? str() +a +#? float() +b + +class Employee: + pass + +# The typing library is not installable for Python 2.6, therefore ignore the +# following tests. +# python >= 2.7 + +from typing import List +x = [] # type: List[Employee] +#? Employee() +x[1] +x, y, z = [], [], [] # type: List[int], List[int], List[str] +#? int() +y[2] +x, y, z = [], [], [] # type: (List[float], List[float], List[BB]) +for zi in z: + #? BB() + zi + +x = [ + 1, + 2, +] # type: List[str] + +#? str() +x[1] + + +for bar in foo(): # type: str + #? str() + bar + +for bar, baz in foo(): # type: int, float + #? int() + bar + #? float() + baz + +for bar, baz in foo(): + # type: str, str + """ type hinting on next line should not work """ + #? + bar + #? + baz + +with foo(): # type: int + ... + +with foo() as f: # type: str + #? str() + f + +with foo() as f: + # type: str + """ type hinting on next line should not work """ + #? + f + +aaa = some_extremely_long_function_name_that_doesnt_leave_room_for_hints() \ + # type: float # We should be able to put hints on the next line with a \ +#? float() +aaa diff --git a/test/completion/pep0484_typing.py b/test/completion/pep0484_typing.py new file mode 100644 index 00000000..75c1c0b0 --- /dev/null +++ b/test/completion/pep0484_typing.py @@ -0,0 +1,263 @@ +""" +Test the typing library, with docstrings. This is needed since annotations +are not supported in python 2.7 else then annotating by comment (and this is +still TODO at 2016-01-23) +""" +# There's no Python 2.6 typing module. +# python >= 2.7 +import typing +class B: + pass + +def we_can_has_sequence(p, q, r, s, t, u): + """ + :type p: typing.Sequence[int] + :type q: typing.Sequence[B] + :type r: typing.Sequence[int] + :type s: typing.Sequence["int"] + :type t: typing.MutableSequence[dict] + :type u: typing.List[float] + """ + #? ["count"] + p.c + #? int() + p[1] + #? ["count"] + q.c + #? B() + q[1] + #? ["count"] + r.c + #? int() + r[1] + #? ["count"] + s.c + #? int() + s[1] + #? [] + s.a + #? ["append"] + t.a + #? dict() + t[1] + #? ["append"] + u.a + #? float() + u[1] + +def iterators(ps, qs, rs, ts): + """ + :type ps: typing.Iterable[int] + :type qs: typing.Iterator[str] + :type rs: typing.Sequence["ForwardReference"] + :type ts: typing.AbstractSet["float"] + """ + for p in ps: + #? int() + p + #? + next(ps) + a, b = ps + #? int() + a + ##? int() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + # test below is just to make sure that in case it gets fixed by accident + # these tests will be fixed as well the way they should be + #? + b + + for q in qs: + #? str() + q + #? str() + next(qs) + for r in rs: + #? ForwardReference() + r + #? + next(rs) + for t in ts: + #? float() + t + +def sets(p, q): + """ + :type p: typing.AbstractSet[int] + :type q: typing.MutableSet[float] + """ + #? [] + p.a + #? ["add"] + q.a + +def tuple(p, q, r): + """ + :type p: typing.Tuple[int] + :type q: typing.Tuple[int, str, float] + :type r: typing.Tuple[B, ...] + """ + #? int() + p[0] + #? int() + q[0] + #? str() + q[1] + #? float() + q[2] + #? B() + r[0] + #? B() + r[1] + #? B() + r[2] + #? B() + r[10000] + i, s, f = q + #? int() + i + ##? str() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? + s + ##? float() --- TODO fix support for tuple assignment + # https://github.com/davidhalter/jedi/pull/663#issuecomment-172317854 + #? + f + +class Key: + pass + +class Value: + pass + +def mapping(p, q, d, r, s, t): + """ + :type p: typing.Mapping[Key, Value] + :type q: typing.MutableMapping[Key, Value] + :type d: typing.Dict[Key, Value] + :type r: typing.KeysView[Key] + :type s: typing.ValuesView[Value] + :type t: typing.ItemsView[Key, Value] + """ + #? [] + p.setd + #? ["setdefault"] + q.setd + #? ["setdefault"] + d.setd + #? Value() + p[1] + for key in p: + #? Key() + key + for key in p.keys(): + #? Key() + key + for value in p.values(): + #? Value() + value + for item in p.items(): + #? Key() + item[0] + #? Value() + item[1] + (key, value) = item + #? Key() + key + #? Value() + value + for key, value in p.items(): + #? Key() + key + #? Value() + value + for key in r: + #? Key() + key + for value in s: + #? Value() + value + for key, value in t: + #? Key() + key + #? Value() + value + +def union(p, q, r, s, t): + """ + :type p: typing.Union[int] + :type q: typing.Union[int, int] + :type r: typing.Union[int, str, "int"] + :type s: typing.Union[int, typing.Union[str, "typing.Union['float', 'dict']"]] + :type t: typing.Union[int, None] + """ + #? int() + p + #? int() + q + #? int() str() + r + #? int() str() float() dict() + s + #? int() + t + +def optional(p): + """ + :type p: typing.Optional[int] + Optional does not do anything special. However it should be recognised + as being of that type. Jedi doesn't do anything with the extra into that + it can be None as well + """ + #? int() + p + +class ForwardReference: + pass + +class TestDict(typing.Dict[str, int]): + def setdud(self): + pass + +def testdict(x): + """ + :type x: TestDict + """ + #? ["setdud", "setdefault"] + x.setd + for key in x.keys(): + #? str() + key + for value in x.values(): + #? int() + value + +x = TestDict() +#? ["setdud", "setdefault"] +x.setd +for key in x.keys(): + #? str() + key +for value in x.values(): + #? int() + value +# python >= 3.2 +""" +docstrings have some auto-import, annotations can use all of Python's +import logic +""" +import typing as t +def union2(x: t.Union[int, str]): + #? int() str() + x + +from typing import Union +def union3(x: Union[int, str]): + #? int() str() + x + +from typing import Union as U +def union4(x: U[int, str]): + #? int() str() + x diff --git a/test/completion/precedence.py b/test/completion/precedence.py index 68f15f43..60781158 100644 --- a/test/completion/precedence.py +++ b/test/completion/precedence.py @@ -48,6 +48,14 @@ a = 3 * "a" #? str() a +a = 3 * "a" +#? str() +a + +#? int() +(3 ** 3) +#? int() str() +(3 ** 'a') # ----------------- # assignments diff --git a/test/completion/stdlib.py b/test/completion/stdlib.py index 6c5a330c..92b7929a 100644 --- a/test/completion/stdlib.py +++ b/test/completion/stdlib.py @@ -35,6 +35,45 @@ next(open('')) #? ['__itemsize__'] tuple.__itemsize__ +# ----------------- +# type() calls with one parameter +# ----------------- +#? int +type(1) +#? int +type(int()) +#? type +type(int) +#? type +type(type) +#? list +type([]) + +def x(): + yield 1 +generator = type(x()) +#? generator +type(x for x in []) +#? type(x) +type(lambda: x) + +import math +import os +#? type(os) +type(math) +class X(): pass +#? type +type(X) + +# ----------------- +# enumerate +# ----------------- +for i, j in enumerate(["as", "ad"]): + #? int() + i + #? str() + j + # ----------------- # re # ----------------- @@ -173,3 +212,13 @@ class B(object): cls = random.choice([A, B]) #? ['say', 'shout'] cls().s + +# ----------------- +# random +# ----------------- + +import zipfile +z = zipfile.ZipFile("foo") +# It's too slow. So we don't run it at the moment. +##? ['upper'] +z.read('name').upper diff --git a/test/completion/types.py b/test/completion/types.py index 8bca1942..768b6602 100644 --- a/test/completion/types.py +++ b/test/completion/types.py @@ -79,6 +79,18 @@ dic2.popitem #? int() dic2['asdf'] +d = {'a': 3, 1.0: list} + +#? int() list +d.values()[0] +##? int() list +dict(d).values()[0] + +#? str() +d.items()[0][0] +#? int() +d.items()[0][1] + # ----------------- # set # ----------------- diff --git a/test/completion/usages.py b/test/completion/usages.py index c5688f77..3805152b 100644 --- a/test/completion/usages.py +++ b/test/completion/usages.py @@ -3,10 +3,10 @@ Renaming tests. This means search for usages. I always leave a little bit of space to add room for additions, because the results always contain position informations. """ -#< 4 (0,4), (3,0), (5,0), (17,0) +#< 4 (0,4), (3,0), (5,0), (17,0), (12,4), (14,5), (15,0) def abc(): pass -#< 0 (-3,4), (0,0), (2,0), (14,0) +#< 0 (-3,4), (0,0), (2,0), (14,0), (9,4), (11,5), (12,0) abc.d.a.bsaasd.abc.d abc @@ -20,7 +20,7 @@ if 1: else: (abc) = abc = -#< (-17,4), (-14,0), (-12,0), (0,0) +#< (-17,4), (-14,0), (-12,0), (0,0), (-2,0), (-3,5), (-5,4) abc abc = 5 @@ -83,7 +83,7 @@ import module_not_exists module_not_exists -#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17), ('imports', 70, 16) +#< ('rename1', 1,0), (0,24), (3,0), (6,17), ('rename2', 4,5), (10,17), (13,17), ('imports', 72, 16) from import_tree import rename1 #< (0,8), ('rename1',3,0), ('rename2',4,20), ('rename2',6,0), (3,32), (7,32), (4,0) @@ -93,7 +93,7 @@ rename1.abc from import_tree.rename1 import abc abc -#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17), ('imports', 70, 16) +#< 20 ('rename1', 1,0), ('rename2', 4,5), (-10,24), (-7,0), (-4,17), (0,17), (3,17), ('imports', 72, 16) from import_tree.rename1 import abc #< (0, 32), diff --git a/test/conftest.py b/test/conftest.py index 7349279a..ac04e268 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,8 +6,10 @@ import pytest from . import helpers from . import run from . import refactor + import jedi from jedi.evaluate.analysis import Warning +from jedi import settings def pytest_addoption(parser): @@ -88,11 +90,15 @@ class StaticAnalysisCase(object): The tests also start with `#!`, like the goto_definition tests. """ def __init__(self, path): - self.skip = False self._path = path with open(path) as f: self._source = f.read() + self.skip = False + for line in self._source.splitlines(): + self.skip = self.skip or run.skip_python_version(line) + + def collect_comparison(self): cases = [] for line_nr, line in enumerate(self._source.splitlines(), 1): @@ -121,5 +127,4 @@ def isolated_jedi_cache(monkeypatch, tmpdir): Same as `clean_jedi_cache`, but create the temporary directory for each test case (scope='function'). """ - from jedi import settings monkeypatch.setattr(settings, 'cache_directory', str(tmpdir)) diff --git a/test/run.py b/test/run.py index a48e1fb2..3eef7343 100755 --- a/test/run.py +++ b/test/run.py @@ -54,7 +54,7 @@ Alternate Test Runner If you don't like the output of ``py.test``, there's an alternate test runner that you can start by running ``./run.py``. The above example could be run by:: - ./run.py basic 4 6 8 + ./run.py basic 4 6 8 50-80 The advantage of this runner is simplicity and more customized error reports. Using both runners will help you to have a quicker overview of what's @@ -111,12 +111,16 @@ Tests look like this:: """ import os import re +import sys +import operator from ast import literal_eval from io import StringIO from functools import reduce import jedi from jedi._compatibility import unicode, is_py3 +from jedi.parser import Parser, load_grammar +from jedi.api.classes import Definition TEST_COMPLETIONS = 0 @@ -127,7 +131,7 @@ TEST_USAGES = 3 class IntegrationTestCase(object): def __init__(self, test_type, correct, line_nr, column, start, line, - path=None): + path=None, skip=None): self.test_type = test_type self.correct = correct self.line_nr = line_nr @@ -135,7 +139,7 @@ class IntegrationTestCase(object): self.start = start self.line = line self.path = path - self.skip = None + self.skip = skip @property def module_name(self): @@ -170,36 +174,35 @@ class IntegrationTestCase(object): return compare_cb(self, comp_str, set(literal_eval(self.correct))) def run_goto_definitions(self, compare_cb): + script = self.script() + evaluator = script._evaluator + def comparison(definition): suffix = '()' if definition.type == 'instance' else '' return definition.desc_with_module + suffix def definition(correct, correct_start, path): - def defs(line_nr, indent): - s = jedi.Script(self.source, line_nr, indent, path) - return set(s.goto_definitions()) - should_be = set() - number = 0 - for index in re.finditer('(?:[^ ]+)', correct): - end = index.end() - # +3 because of the comment start `#? ` - end += 3 - number += 1 - try: - should_be |= defs(self.line_nr - 1, end + correct_start) - except Exception: - print('could not resolve %s indent %s' - % (self.line_nr - 1, end)) - raise - # because the objects have different ids, `repr`, then compare. + for match in re.finditer('(?:[^ ]+)', correct): + string = match.group(0) + parser = Parser(load_grammar(), string, start_symbol='eval_input') + parser.position_modifier.line = self.line_nr + element = parser.get_parsed_node() + element.parent = jedi.api.completion.get_user_scope( + script._get_module(), + (self.line_nr, self.column) + ) + results = evaluator.eval_element(element) + if not results: + raise Exception('Could not resolve %s on line %s' + % (match.string, self.line_nr - 1)) + + should_be |= set(Definition(evaluator, r) for r in results) + + # Because the objects have different ids, `repr`, then compare. should = set(comparison(r) for r in should_be) - if len(should) < number: - raise Exception('Solution @%s not right, too few test results: %s' - % (self.line_nr - 1, should)) return should - script = self.script() should = definition(self.correct, self.start, script.path) result = script.goto_definitions() is_str = set(comparison(r) for r in result) @@ -232,12 +235,35 @@ class IntegrationTestCase(object): return compare_cb(self, compare, sorted(wanted)) -def collect_file_tests(lines, lines_to_execute): - makecase = lambda t: IntegrationTestCase(t, correct, line_nr, column, - start, line) +def skip_python_version(line): + comp_map = { + '==': 'eq', + '<=': 'le', + '>=': 'ge', + '<': 'lt', + '>': 'gt', + } + # check for python minimal version number + match = re.match(r" *# *python *([<>]=?|==) *(\d+(?:\.\d+)?)$", line) + if match: + minimal_python_version = tuple( + map(int, match.group(2).split("."))) + operation = getattr(operator, comp_map[match.group(1)]) + if not operation(sys.version_info, minimal_python_version): + return "Minimal python version %s %s" % (match.group(1), match.group(2)) + + return None + + +def collect_file_tests(path, lines, lines_to_execute): + def makecase(t): + return IntegrationTestCase(t, correct, line_nr, column, + start, line, path=path, skip=skip) + start = None correct = None test_type = None + skip = None for line_nr, line in enumerate(lines, 1): if correct is not None: r = re.match('^(\d+)\s*(.*)$', correct) @@ -257,6 +283,7 @@ def collect_file_tests(lines, lines_to_execute): yield makecase(TEST_DEFINITIONS) correct = None else: + skip = skip or skip_python_version(line) try: r = re.search(r'(?:^|(?<=\s))#([?!<])\s*([^\n]*)', line) # test_type is ? for completion and ! for goto_assignments @@ -269,9 +296,14 @@ def collect_file_tests(lines, lines_to_execute): except AttributeError: correct = None else: - # skip the test, if this is not specified test - if lines_to_execute and line_nr not in lines_to_execute: - correct = None + # Skip the test, if this is not specified test. + for l in lines_to_execute: + if isinstance(l, tuple) and l[0] <= line_nr <= l[1] \ + or line_nr == l: + break + else: + if lines_to_execute: + correct = None def collect_dir_tests(base_dir, test_files, check_thirdparty=False): @@ -290,12 +322,14 @@ def collect_dir_tests(base_dir, test_files, check_thirdparty=False): skip = 'Thirdparty-Library %s not found.' % lib path = os.path.join(base_dir, f_name) - source = open(path).read() - if not is_py3: - source = unicode(source, 'UTF-8') - for case in collect_file_tests(StringIO(source), + + if is_py3: + source = open(path, encoding='utf-8').read() + else: + source = unicode(open(path).read(), 'UTF-8') + + for case in collect_file_tests(path, StringIO(source), lines_to_execute): - case.path = path case.source = source if skip: case.skip = skip @@ -335,7 +369,11 @@ if __name__ == '__main__': test_files = {} last = None for arg in arguments['']: - if arg.isdigit(): + match = re.match('(\d+)-(\d+)', arg) + if match: + start, end = match.groups() + test_files[last].append((int(start), int(end))) + elif arg.isdigit(): if last is None: continue test_files[last].append(int(arg)) @@ -344,7 +382,9 @@ if __name__ == '__main__': last = arg # completion tests: - completion_test_dir = '../test/completion' + dir_ = os.path.dirname(os.path.realpath(__file__)) + completion_test_dir = os.path.join(dir_, '../test/completion') + completion_test_dir = os.path.abspath(completion_test_dir) summary = [] tests_fail = 0 @@ -371,6 +411,8 @@ if __name__ == '__main__': current = cases[0].path if cases else None count = fails = 0 for c in cases: + if c.skip: + continue if current != c.path: file_change(current, count, fails) current = c.path diff --git a/test/static_analysis/attribute_warnings.py b/test/static_analysis/attribute_warnings.py index d31058cc..0e1e5e95 100644 --- a/test/static_analysis/attribute_warnings.py +++ b/test/static_analysis/attribute_warnings.py @@ -35,7 +35,7 @@ Inherited().undefined class SetattrCls(): def __init__(self, dct): # Jedi doesn't even try to understand such code - for k, v in dct: + for k, v in dct.items(): setattr(self, k, v) self.defined = 3 diff --git a/test/static_analysis/branches.py b/test/static_analysis/branches.py new file mode 100644 index 00000000..6828d33c --- /dev/null +++ b/test/static_analysis/branches.py @@ -0,0 +1,46 @@ +# ----------------- +# Simple tests +# ----------------- + +import random + +if random.choice([0, 1]): + x = '' +else: + x = 1 +if random.choice([0, 1]): + y = '' +else: + y = 1 + +# A simple test +if x != 1: + x.upper() +else: + #! 2 attribute-error + x.upper() + pass + +# This operation is wrong, because the types could be different. +#! 6 type-error-operation +z = x + y +# However, here we have correct types. +if x == y: + z = x + y +else: + #! 6 type-error-operation + z = x + y + +# ----------------- +# With a function +# ----------------- + +def addition(a, b): + if type(a) == type(b): + return a + b + else: + #! 9 type-error-operation + return a + b + +addition(1, 1) +addition(1.0, '') diff --git a/test/static_analysis/builtins.py b/test/static_analysis/builtins.py new file mode 100644 index 00000000..86caca65 --- /dev/null +++ b/test/static_analysis/builtins.py @@ -0,0 +1,11 @@ +# ---------- +# isinstance +# ---------- + +isinstance(1, int) +isinstance(1, (int, str)) + +#! 14 type-error-isinstance +isinstance(1, 1) +#! 14 type-error-isinstance +isinstance(1, [int, str]) diff --git a/test/static_analysis/class_simple.py b/test/static_analysis/class_simple.py new file mode 100644 index 00000000..3f84fde0 --- /dev/null +++ b/test/static_analysis/class_simple.py @@ -0,0 +1,13 @@ +class Base(object): + class Nested(): + def foo(): + pass + + +class X(Base.Nested): + pass + + +X().foo() +#! 4 attribute-error +X().bar() diff --git a/test/static_analysis/comprehensions.py b/test/static_analysis/comprehensions.py new file mode 100644 index 00000000..4af799d8 --- /dev/null +++ b/test/static_analysis/comprehensions.py @@ -0,0 +1,41 @@ +[a + 1 for a in [1, 2]] + +#! 3 type-error-operation +[a + '' for a in [1, 2]] +#! 3 type-error-operation +(a + '' for a in [1, 2]) + +#! 12 type-error-not-iterable +[a for a in 1] + +tuple(str(a) for a in [1]) + +#! 8 type-error-operation +tuple(a + 3 for a in ['']) + +# ---------- +# Some variables within are not defined +# ---------- + +#! 12 name-error +[1 for a in NOT_DEFINFED for b in a if 1] + +#! 25 name-error +[1 for a in [1] for b in NOT_DEFINED if 1] + +#! 12 name-error +[1 for a in NOT_DEFINFED for b in [1] if 1] + +#! 19 name-error +(1 for a in [1] if NOT_DEFINED) + +# ---------- +# unbalanced sides. +# ---------- + +# ok +(1 for a, b in [(1, 2)]) +#! 13 value-error-too-few-values +(1 for a, b, c in [(1, 2)]) +#! 10 value-error-too-many-values +(1 for a, b in [(1, 2, 3)]) diff --git a/test/static_analysis/generators.py b/test/static_analysis/generators.py index 6da66079..b9418002 100644 --- a/test/static_analysis/generators.py +++ b/test/static_analysis/generators.py @@ -1,7 +1,7 @@ def generator(): yield 1 -#! 12 type-error-generator +#! 11 type-error-not-subscriptable generator()[0] list(generator())[0] diff --git a/test/static_analysis/iterable.py b/test/static_analysis/iterable.py new file mode 100644 index 00000000..0eae367d --- /dev/null +++ b/test/static_analysis/iterable.py @@ -0,0 +1,21 @@ + +a, b = {'asdf': 3, 'b': 'str'} +a + +x = [1] +x[0], b = {'a': 1, 'b': '2'} + +dct = {3: ''} +for x in dct: + pass + +#! 4 type-error-not-iterable +for x, y in dct: + pass + +# Shouldn't cause issues, because if there are no types (or we don't know what +# the types are, we should just ignore it. +#! 0 value-error-too-few-values +a, b = [] +#! 7 name-error +a, b = NOT_DEFINED diff --git a/test/static_analysis/keywords.py b/test/static_analysis/keywords.py new file mode 100644 index 00000000..e3fcaa43 --- /dev/null +++ b/test/static_analysis/keywords.py @@ -0,0 +1,7 @@ +def raises(): + raise KeyError() + + +def wrong_name(): + #! 6 name-error + raise NotExistingException() diff --git a/test/static_analysis/operations.py b/test/static_analysis/operations.py index 71fbd230..05e1406c 100644 --- a/test/static_analysis/operations.py +++ b/test/static_analysis/operations.py @@ -9,3 +9,8 @@ -1 - int() int() - float() float() - 3.0 + +a = 3 +b = '' +#! 2 type-error-operation +a + b diff --git a/test/static_analysis/python2.py b/test/static_analysis/python2.py new file mode 100644 index 00000000..4d896e3e --- /dev/null +++ b/test/static_analysis/python2.py @@ -0,0 +1,11 @@ +""" +Some special cases of Python 2. +""" +# python <= 2.7 + +# print is syntax: +print 1 +print(1) + +#! 6 name-error +print NOT_DEFINED diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 926f41a2..fc961872 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -7,6 +7,7 @@ from textwrap import dedent from jedi import api from jedi._compatibility import is_py3 from pytest import raises +from jedi.parser import utils def test_preload_modules(): @@ -16,16 +17,15 @@ def test_preload_modules(): for i in modules: assert [i in k for k in parser_cache.keys() if k is not None] - from jedi import cache - temp_cache, cache.parser_cache = cache.parser_cache, {} - parser_cache = cache.parser_cache + temp_cache, utils.parser_cache = utils.parser_cache, {} + parser_cache = utils.parser_cache api.preload_module('sys') check_loaded() # compiled (c_builtin) modules shouldn't be in the cache. api.preload_module('json', 'token') check_loaded('json', 'token') - cache.parser_cache = temp_cache + utils.parser_cache = temp_cache def test_empty_script(): @@ -62,7 +62,8 @@ def _check_number(source, result='float'): def test_completion_on_number_literals(): # No completions on an int literal (is a float). - assert api.Script('1.').completions() == [] + assert [c.name for c in api.Script('1.').completions()] \ + == ['and', 'if', 'in', 'is', 'not', 'or'] # Multiple points after an int literal basically mean that there's a float # and a call after that. @@ -84,7 +85,6 @@ def test_completion_on_hex_literals(): # (invalid statements). assert api.Script('0b2.').completions() == [] _check_number('0b1.', 'int') # binary - _check_number('0o7.', 'int') # octal _check_number('0x2e.', 'int') _check_number('0xE7.', 'int') diff --git a/test/test_api/test_call_signatures.py b/test/test_api/test_call_signatures.py index 46d55e76..c238ca56 100644 --- a/test/test_api/test_call_signatures.py +++ b/test/test_api/test_call_signatures.py @@ -7,29 +7,29 @@ from jedi import cache from jedi._compatibility import is_py33 +def assert_signature(source, expected_name, expected_index=0, line=None, column=None): + signatures = Script(source, line, column).call_signatures() + + assert len(signatures) <= 1 + + if not signatures: + assert expected_name is None, \ + 'There are no signatures, but `%s` expected.' % expected_name + else: + assert signatures[0].name == expected_name + assert signatures[0].index == expected_index + return signatures[0] + + class TestCallSignatures(TestCase): - def _run(self, source, expected_name, expected_index=0, line=None, column=None): - signatures = Script(source, line, column).call_signatures() - - assert len(signatures) <= 1 - - if not signatures: - assert expected_name is None - else: - assert signatures[0].name == expected_name - assert signatures[0].index == expected_index - def _run_simple(self, source, name, index=0, column=None, line=1): - self._run(source, name, index, line, column) + assert_signature(source, name, index, line, column) def test_valid_call(self): - self._run('str()', 'str', column=4) + assert_signature('str()', 'str', column=4) def test_simple(self): run = self._run_simple - s7 = "str().upper().center(" - s8 = "str(int[zip(" - run(s7, 'center', 0) # simple s1 = "sorted(a, str(" @@ -48,13 +48,16 @@ class TestCallSignatures(TestCase): run(s3, None, column=5) run(s3, None) - # more complicated + def test_more_complicated(self): + run = self._run_simple + ''' s4 = 'abs(zip(), , set,' run(s4, None, column=3) run(s4, 'abs', 0, 4) run(s4, 'zip', 0, 8) run(s4, 'abs', 0, 9) #run(s4, 'abs', 1, 10) + ''' s5 = "sorted(1,\nif 2:\n def a():" run(s5, 'sorted', 0, 7) @@ -72,10 +75,11 @@ class TestCallSignatures(TestCase): run("import time; abc = time; abc.sleep(", 'sleep', 0) + def test_issue_57(self): # jedi #57 s = "def func(alpha, beta): pass\n" \ "func(alpha='101'," - run(s, 'func', 0, column=13, line=2) + self._run_simple(s, 'func', 0, column=13, line=2) def test_flows(self): # jedi-vim #9 @@ -96,14 +100,14 @@ class TestCallSignatures(TestCase): if 1: pass """ - self._run(s, 'abc', 0, line=6, column=24) + assert_signature(s, 'abc', 0, line=6, column=24) s = """ import re def huhu(it): re.compile( return it * 2 """ - self._run(s, 'compile', 0, line=4, column=31) + assert_signature(s, 'compile', 0, line=4, column=31) # jedi-vim #70 s = """def foo(""" @@ -111,7 +115,7 @@ class TestCallSignatures(TestCase): # jedi-vim #116 s = """import itertools; test = getattr(itertools, 'chain'); test(""" - self._run(s, 'chain', 0) + assert_signature(s, 'chain', 0) def test_call_signature_on_module(self): """github issue #240""" @@ -124,7 +128,7 @@ class TestCallSignatures(TestCase): def f(a, b): pass f( )""") - self._run(s, 'f', 0, line=3, column=3) + assert_signature(s, 'f', 0, line=3, column=3) def test_multiple_signatures(self): s = dedent("""\ @@ -143,7 +147,7 @@ class TestCallSignatures(TestCase): def x(): pass """) - self._run(s, 'abs', 0, line=1, column=5) + assert_signature(s, 'abs', 0, line=1, column=5) def test_decorator_in_class(self): """ @@ -169,28 +173,27 @@ class TestCallSignatures(TestCase): assert x == ['*args'] def test_additional_brackets(self): - self._run('str((', 'str', 0) + assert_signature('str((', 'str', 0) def test_unterminated_strings(self): - self._run('str(";', 'str', 0) + assert_signature('str(";', 'str', 0) def test_whitespace_before_bracket(self): - self._run('str (', 'str', 0) - self._run('str (";', 'str', 0) - # TODO this is not actually valid Python, the newline token should be - # ignored. - self._run('str\n(', 'str', 0) + assert_signature('str (', 'str', 0) + assert_signature('str (";', 'str', 0) + assert_signature('str\n(', None) def test_brackets_in_string_literals(self): - self._run('str (" (', 'str', 0) - self._run('str (" )', 'str', 0) + assert_signature('str (" (', 'str', 0) + assert_signature('str (" )', 'str', 0) def test_function_definitions_should_break(self): """ Function definitions (and other tokens that cannot exist within call signatures) should break and not be able to return a call signature. """ - assert not Script('str(\ndef x').call_signatures() + assert_signature('str(\ndef x', 'str', 0) + assert not Script('str(\ndef x(): pass').call_signatures() def test_flow_call(self): assert not Script('if (1').call_signatures() @@ -207,14 +210,14 @@ class TestCallSignatures(TestCase): A().test1().test2(''') - self._run(source, 'test2', 0) + assert_signature(source, 'test2', 0) def test_return(self): source = dedent(''' def foo(): return '.'.join()''') - self._run(source, 'join', 0, column=len(" return '.'.join(")) + assert_signature(source, 'join', 0, column=len(" return '.'.join(")) class TestParams(TestCase): @@ -249,7 +252,6 @@ class TestParams(TestCase): assert p[0].name == 'suffix' - def test_signature_is_definition(): """ Through inheritance, a call signature is a sub class of Definition. @@ -257,7 +259,7 @@ def test_signature_is_definition(): """ s = """class Spam(): pass\nSpam""" signature = Script(s + '(').call_signatures()[0] - definition = Script(s + '(').goto_definitions()[0] + definition = Script(s + '(', column=0).goto_definitions()[0] signature.line == 1 signature.column == 6 @@ -315,13 +317,23 @@ def test_completion_interference(): assert Script('open(').call_signatures() -def test_signature_index(): - def get(source): - return Script(source).call_signatures()[0] +def test_keyword_argument_index(): + def get(source, column=None): + return Script(source, column=column).call_signatures()[0] assert get('sorted([], key=a').index == 2 + assert get('sorted([], key=').index == 2 assert get('sorted([], no_key=a').index is None + kw_func = 'def foo(a, b): pass\nfoo(b=3, a=4)' + assert get(kw_func, column=len('foo(b')).index == 0 + assert get(kw_func, column=len('foo(b=')).index == 1 + assert get(kw_func, column=len('foo(b=3, a=')).index == 0 + + kw_func_simple = 'def foo(a, b): pass\nfoo(b=4)' + assert get(kw_func_simple, column=len('foo(b')).index == 0 + assert get(kw_func_simple, column=len('foo(b=')).index == 1 + args_func = 'def foo(*kwargs): pass\n' assert get(args_func + 'foo(a').index == 0 assert get(args_func + 'foo(a, b').index == 0 @@ -333,6 +345,7 @@ def test_signature_index(): both = 'def foo(*args, **kwargs): pass\n' assert get(both + 'foo(a=2').index == 1 assert get(both + 'foo(a=2, b=2').index == 1 + assert get(both + 'foo(a=2, b=2)', column=len('foo(b=2, a=2')).index == 1 assert get(both + 'foo(a, b, c').index == 0 @@ -343,3 +356,24 @@ def test_bracket_start(): return signatures[0].bracket_start assert bracket_start('str(') == (1, 3) + + +def test_different_caller(): + """ + It's possible to not use names, but another function result or an array + index and then get the call signature of it. + """ + + assert_signature('[str][0](', 'str', 0) + assert_signature('[str][0]()', 'str', 0, column=len('[str][0](')) + + assert_signature('(str)(', 'str', 0) + assert_signature('(str)()', 'str', 0, column=len('(str)(')) + + +def test_in_function(): + code = dedent('''\ + class X(): + @property + def func(''') + assert not Script(code).call_signatures() diff --git a/test/test_api/test_classes.py b/test/test_api/test_classes.py index 59ce2837..64d92648 100644 --- a/test/test_api/test_classes.py +++ b/test/test_api/test_classes.py @@ -316,7 +316,7 @@ class TestGotoAssignments(TestCase): n = nms[1].goto_assignments()[0] # This is very special, normally the name doesn't chance, but since # os.path is a sys.modules hack, it does. - assert n.name in ('ntpath', 'posixpath') + assert n.name in ('ntpath', 'posixpath', 'os2emxpath') assert n.type == 'module' def test_import_alias(self): @@ -333,3 +333,22 @@ class TestGotoAssignments(TestCase): assert len(ass) == 1 assert ass[0].name == 'json' assert ass[0].type == 'module' + + +def test_added_equals_to_params(): + def run(rest_source): + source = dedent(""" + def foo(bar, baz): + pass + """) + results = Script(source + rest_source).completions() + assert len(results) == 1 + return results[0] + + assert run('foo(bar').name_with_symbols == 'bar=' + assert run('foo(bar').complete == '=' + assert run('foo(bar, baz').complete == '=' + assert run(' bar').name_with_symbols == 'bar' + assert run(' bar').complete == '' + x = run('foo(bar=isins').name_with_symbols + assert x == 'isinstance' diff --git a/test/test_api/test_defined_names.py b/test/test_api/test_defined_names.py index 3eb41356..aae1f6ca 100644 --- a/test/test_api/test_defined_names.py +++ b/test/test_api/test_defined_names.py @@ -2,19 +2,19 @@ Tests for `api.defined_names`. """ -import textwrap +from textwrap import dedent -from jedi import api +from jedi import defined_names, names from ..helpers import TestCase class TestDefinedNames(TestCase): - def assert_definition_names(self, definitions, names): - assert [d.name for d in definitions] == names + def assert_definition_names(self, definitions, names_): + assert [d.name for d in definitions] == names_ - def check_defined_names(self, source, names): - definitions = api.names(textwrap.dedent(source)) - self.assert_definition_names(definitions, names) + def check_defined_names(self, source, names_): + definitions = names(dedent(source)) + self.assert_definition_names(definitions, names_) return definitions def test_get_definitions_flat(self): @@ -76,7 +76,17 @@ class TestDefinedNames(TestCase): def test_follow_imports(): # github issue #344 - imp = api.defined_names('import datetime')[0] + imp = defined_names('import datetime')[0] assert imp.name == 'datetime' datetime_names = [str(d.name) for d in imp.defined_names()] assert 'timedelta' in datetime_names + + +def test_names_twice(): + source = dedent(''' + def lol(): + pass + ''') + + defs = names(source=source) + assert defs[0].defined_names() == [] diff --git a/test/test_api/test_interpreter.py b/test/test_api/test_interpreter.py index ef2262d4..7c82ea58 100644 --- a/test/test_api/test_interpreter.py +++ b/test/test_api/test_interpreter.py @@ -5,6 +5,64 @@ Tests of ``jedi.api.Interpreter``. from ..helpers import TestCase import jedi from jedi._compatibility import is_py33 +from jedi.evaluate.compiled import mixed + +class _GlobalNameSpace(): + class SideEffectContainer(): + pass + + +def get_completion(source, namespace): + i = jedi.Interpreter(source, [namespace]) + completions = i.completions() + assert len(completions) == 1 + return completions[0] + + +def test_builtin_details(): + import keyword + + class EmptyClass: + pass + + variable = EmptyClass() + + def func(): + pass + + cls = get_completion('EmptyClass', locals()) + var = get_completion('variable', locals()) + f = get_completion('func', locals()) + m = get_completion('keyword', locals()) + assert cls.type == 'class' + assert var.type == 'instance' + assert f.type == 'function' + assert m.type == 'module' + + +def test_nested_resolve(): + class XX(): + def x(): + pass + + cls = get_completion('XX', locals()) + func = get_completion('XX.x', locals()) + assert func.start_pos == (cls.start_pos[0] + 1, 12) + + +def test_side_effect_completion(): + """ + In the repl it's possible to cause side effects that are not documented in + Python code, however we want references to Python code as well. Therefore + we need some mixed kind of magic for tests. + """ + _GlobalNameSpace.SideEffectContainer.foo = 1 + side_effect = get_completion('SideEffectContainer', _GlobalNameSpace.__dict__) + + # It's a class that contains MixedObject. + assert isinstance(side_effect._definition.base, mixed.MixedObject) + foo = get_completion('SideEffectContainer.foo', _GlobalNameSpace.__dict__) + assert foo.name == 'foo' class TestInterpreterAPI(TestCase): @@ -17,19 +75,19 @@ class TestInterpreterAPI(TestCase): def test_complete_raw_function(self): from os.path import join - self.check_interpreter_complete('join().up', + self.check_interpreter_complete('join("").up', locals(), ['upper']) def test_complete_raw_function_different_name(self): from os.path import join as pjoin - self.check_interpreter_complete('pjoin().up', + self.check_interpreter_complete('pjoin("").up', locals(), ['upper']) def test_complete_raw_module(self): import os - self.check_interpreter_complete('os.path.join().up', + self.check_interpreter_complete('os.path.join("a").up', locals(), ['upper']) diff --git a/test/test_api/test_usages.py b/test/test_api/test_usages.py new file mode 100644 index 00000000..1325e412 --- /dev/null +++ b/test/test_api/test_usages.py @@ -0,0 +1,6 @@ +import jedi + + +def test_import_usage(): + s = jedi.Script("from .. import foo", line=1, column=18, path="foo.py") + assert [usage.line for usage in s.usages()] == [1] diff --git a/test/test_cache.py b/test/test_cache.py index b1dfb709..7cff4d4c 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -8,7 +8,7 @@ import pytest import jedi from jedi import settings, cache -from jedi.cache import ParserCacheItem, ParserPickling +from jedi.parser.utils import ParserCacheItem, ParserPickling ParserPicklingCls = type(ParserPickling) diff --git a/test/test_evaluate/buildout_project/bin/binary_file b/test/test_evaluate/buildout_project/bin/binary_file new file mode 100644 index 00000000..f1ad7558 --- /dev/null +++ b/test/test_evaluate/buildout_project/bin/binary_file @@ -0,0 +1 @@ +‰PNG diff --git a/test/test_evaluate/test_absolute_import.py b/test/test_evaluate/test_absolute_import.py index a453a8aa..a45bffa5 100644 --- a/test/test_evaluate/test_absolute_import.py +++ b/test/test_evaluate/test_absolute_import.py @@ -4,7 +4,7 @@ Python 2.X) """ import jedi from jedi._compatibility import u -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from .. import helpers @@ -12,7 +12,7 @@ def test_explicit_absolute_imports(): """ Detect modules with ``from __future__ import absolute_import``. """ - parser = Parser(load_grammar(), u("from __future__ import absolute_import"), "test.py") + parser = ParserWithRecovery(load_grammar(), u("from __future__ import absolute_import"), "test.py") assert parser.module.has_explicit_absolute_import @@ -20,7 +20,7 @@ def test_no_explicit_absolute_imports(): """ Detect modules without ``from __future__ import absolute_import``. """ - parser = Parser(load_grammar(), u("1"), "test.py") + parser = ParserWithRecovery(load_grammar(), u("1"), "test.py") assert not parser.module.has_explicit_absolute_import @@ -30,7 +30,7 @@ def test_dont_break_imports_without_namespaces(): assume that all imports have non-``None`` namespaces. """ src = u("from __future__ import absolute_import\nimport xyzzy") - parser = Parser(load_grammar(), src, "test.py") + parser = ParserWithRecovery(load_grammar(), src, "test.py") assert parser.module.has_explicit_absolute_import diff --git a/test/test_evaluate/test_annotations.py b/test/test_evaluate/test_annotations.py index 1fefde3c..67fe84e1 100644 --- a/test/test_evaluate/test_annotations.py +++ b/test/test_evaluate/test_annotations.py @@ -8,8 +8,8 @@ import pytest def test_simple_annotations(): """ Annotations only exist in Python 3. - At the moment we ignore them. So they should be parsed and not interfere - with anything. + If annotations adhere to PEP-0484, we use them (they override inference), + else they are parsed but ignored """ source = dedent("""\ @@ -27,3 +27,34 @@ def test_simple_annotations(): annot_ret('')""") assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['str'] + + source = dedent("""\ + def annot(a:int): + return a + + annot('')""") + + assert [d.name for d in jedi.Script(source, ).goto_definitions()] == ['int'] + + +@pytest.mark.skipif('sys.version_info[0] < 3') +@pytest.mark.parametrize('reference', [ + 'assert 1', + '1', + 'def x(): pass', + '1, 2', + r'1\n' +]) +def test_illegal_forward_references(reference): + source = 'def foo(bar: "%s"): bar' % reference + + assert not jedi.Script(source).goto_definitions() + + +@pytest.mark.skipif('sys.version_info[0] < 3') +def test_lambda_forward_references(): + source = 'def foo(bar: "lambda: 3"): bar' + + # For now just receiving the 3 is ok. I'm doubting that this is what we + # want. We also execute functions. Should we only execute classes? + assert jedi.Script(source).goto_definitions() diff --git a/test/test_evaluate/test_buildout_detection.py b/test/test_evaluate/test_buildout_detection.py index f3164a7d..c5c65568 100644 --- a/test/test_evaluate/test_buildout_detection.py +++ b/test/test_evaluate/test_buildout_detection.py @@ -7,7 +7,7 @@ from jedi.evaluate.sys_path import (_get_parent_dir_with_file, sys_path_with_modifications, _check_module) from jedi.evaluate import Evaluator -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from ..helpers import cwd_at @@ -37,7 +37,7 @@ def test_append_on_non_sys_path(): d = Dummy() d.path.append('foo')""")) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) paths = _check_module(Evaluator(grammar), p.module) assert len(paths) > 0 assert 'foo' not in paths @@ -48,7 +48,7 @@ def test_path_from_invalid_sys_path_assignment(): import sys sys.path = 'invalid'""")) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) paths = _check_module(Evaluator(grammar), p.module) assert len(paths) > 0 assert 'invalid' not in paths @@ -60,7 +60,7 @@ def test_sys_path_with_modifications(): import os """)) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) p.module.path = os.path.abspath(os.path.join(os.curdir, 'module_name.py')) paths = sys_path_with_modifications(Evaluator(grammar), p.module) assert '/tmp/.buildout/eggs/important_package.egg' in paths @@ -83,7 +83,7 @@ def test_path_from_sys_path_assignment(): if __name__ == '__main__': sys.exit(important_package.main())""")) grammar = load_grammar() - p = Parser(grammar, SRC) + p = ParserWithRecovery(grammar, SRC) paths = _check_module(Evaluator(grammar), p.module) assert 1 not in paths assert '/home/test/.buildout/eggs/important_package.egg' in paths diff --git a/test/test_evaluate/test_compiled.py b/test/test_evaluate/test_compiled.py index 2a6b28dc..7efe64d6 100644 --- a/test/test_evaluate/test_compiled.py +++ b/test/test_evaluate/test_compiled.py @@ -6,31 +6,33 @@ from jedi.evaluate import Evaluator from jedi import Script +def _evaluator(): + return Evaluator(load_grammar()) + + def test_simple(): - e = Evaluator(load_grammar()) - bltn = compiled.CompiledObject(builtins) - obj = compiled.CompiledObject('_str_', bltn) + e = _evaluator() + bltn = compiled.CompiledObject(e, builtins) + obj = compiled.CompiledObject(e, '_str_', bltn) upper = e.find_types(obj, 'upper') assert len(upper) == 1 - objs = list(e.execute(upper[0])) + objs = list(e.execute(list(upper)[0])) assert len(objs) == 1 assert isinstance(objs[0], representation.Instance) def test_fake_loading(): - assert isinstance(compiled.create(Evaluator(load_grammar()), next), Function) + e = _evaluator() + assert isinstance(compiled.create(e, next), Function) - string = compiled.builtin.get_subscope_by_name('str') - from_name = compiled._create_from_name( - compiled.builtin, - string, - '__init__' - ) + builtin = compiled.get_special_object(e, 'BUILTINS') + string = builtin.get_subscope_by_name('str') + from_name = compiled._create_from_name(e, builtin, string, '__init__') assert isinstance(from_name, Function) def test_fake_docstr(): - assert compiled.create(Evaluator(load_grammar()), next).raw_doc == next.__doc__ + assert compiled.create(_evaluator(), next).raw_doc == next.__doc__ def test_parse_function_doc_illegal_docstr(): @@ -47,13 +49,13 @@ def test_doc(): Even CompiledObject docs always return empty docstrings - not None, that's just a Jedi API definition. """ - obj = compiled.CompiledObject(''.__getnewargs__) + obj = compiled.CompiledObject(_evaluator(), ''.__getnewargs__) assert obj.doc == '' def test_string_literals(): def typ(string): - d = Script(string).goto_definitions()[0] + d = Script("a = %s; a" % string).goto_definitions()[0] return d.name assert typ('""') == 'str' diff --git a/test/test_evaluate/test_helpers.py b/test/test_evaluate/test_helpers.py new file mode 100644 index 00000000..02e6d36b --- /dev/null +++ b/test/test_evaluate/test_helpers.py @@ -0,0 +1,16 @@ +from textwrap import dedent + +from jedi import names +from jedi.evaluate import helpers + + +def test_call_of_leaf_in_brackets(): + s = dedent(""" + x = 1 + type(x) + """) + last_x = names(s, references=True, definitions=False)[-1] + name = last_x._name + + call = helpers.call_of_leaf(name) + assert call == name diff --git a/test/test_evaluate/test_imports.py b/test/test_evaluate/test_imports.py index a8825287..98c8baf2 100644 --- a/test/test_evaluate/test_imports.py +++ b/test/test_evaluate/test_imports.py @@ -56,6 +56,13 @@ def test_not_importable_file(): assert not jedi.Script(src, path='example.py').completions() +def test_import_unique(): + src = "import os; os.path" + defs = jedi.Script(src, path='example.py').goto_definitions() + defs = [d._definition for d in defs] + assert len(defs) == len(set(defs)) + + def test_cache_works_with_sys_path_param(tmpdir): foo_path = tmpdir.join('foo') bar_path = tmpdir.join('bar') @@ -70,3 +77,15 @@ def test_cache_works_with_sys_path_param(tmpdir): assert 'bar' in [c.name for c in bar_completions] assert 'foo' not in [c.name for c in bar_completions] + + +def test_import_completion_docstring(): + import abc + s = jedi.Script('"""test"""\nimport ab') + completions = s.completions() + assert len(completions) == 1 + assert completions[0].docstring(fast=False) == abc.__doc__ + + # However for performance reasons not all modules are loaded and the + # docstring is empty in this case. + assert completions[0].docstring() == '' diff --git a/test/test_evaluate/test_precedence.py b/test/test_evaluate/test_precedence.py new file mode 100644 index 00000000..462b7d4a --- /dev/null +++ b/test/test_evaluate/test_precedence.py @@ -0,0 +1,20 @@ +from jedi.parser import load_grammar, Parser +from jedi.evaluate import Evaluator +from jedi.evaluate.compiled import CompiledObject + +import pytest + + +@pytest.mark.skipif('sys.version_info[0] < 3') # Ellipsis does not exists in 2 +@pytest.mark.parametrize('source', [ + '1 == 1', + '1.0 == 1', + '... == ...' +]) +def test_equals(source): + evaluator = Evaluator(load_grammar()) + node = Parser(load_grammar(), source, 'eval_input').get_parsed_node() + results = evaluator.eval_element(node) + assert len(results) == 1 + first = results.pop() + assert isinstance(first, CompiledObject) and first.obj is True diff --git a/test/test_evaluate/test_representation.py b/test/test_evaluate/test_representation.py index 315ba63e..1d3d5622 100644 --- a/test/test_evaluate/test_representation.py +++ b/test/test_evaluate/test_representation.py @@ -32,5 +32,5 @@ def test_class_mro(): pass X""" cls, evaluator = get_definition_and_evaluator(s) - mro = cls.py__mro__(evaluator) + mro = cls.py__mro__() assert [str(c.name) for c in mro] == ['X', 'object'] diff --git a/test/test_evaluate/test_sys_path.py b/test/test_evaluate/test_sys_path.py index 3c44a99c..f7ce0fab 100644 --- a/test/test_evaluate/test_sys_path.py +++ b/test/test_evaluate/test_sys_path.py @@ -5,22 +5,22 @@ import sys import pytest from jedi._compatibility import unicode -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.evaluate import sys_path, Evaluator def test_paths_from_assignment(): def paths(src): grammar = load_grammar() - stmt = Parser(grammar, unicode(src)).module.statements[0] - return list(sys_path._paths_from_assignment(Evaluator(grammar), stmt)) + stmt = ParserWithRecovery(grammar, unicode(src)).module.statements[0] + return set(sys_path._paths_from_assignment(Evaluator(grammar), stmt)) - assert paths('sys.path[0:0] = ["a"]') == ['a'] - assert paths('sys.path = ["b", 1, x + 3, y, "c"]') == ['b', 'c'] - assert paths('sys.path = a = ["a"]') == ['a'] + assert paths('sys.path[0:0] = ["a"]') == set(['a']) + assert paths('sys.path = ["b", 1, x + 3, y, "c"]') == set(['b', 'c']) + assert paths('sys.path = a = ["a"]') == set(['a']) # Fail for complicated examples. - assert paths('sys.path, other = ["a"], 2') == [] + assert paths('sys.path, other = ["a"], 2') == set() # Currently venv site-packages resolution only seeks pythonX.Y/site-packages diff --git a/test/test_integration.py b/test/test_integration.py index edf6ab62..edca7114 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -40,7 +40,10 @@ def test_completion(case, monkeypatch): def test_static_analysis(static_analysis_case): - static_analysis_case.run(assert_static_analysis) + if static_analysis_case.skip is not None: + pytest.skip(static_analysis_case.skip) + else: + static_analysis_case.run(assert_static_analysis) def test_refactor(refactor_case): diff --git a/test/test_integration_import.py b/test/test_integration_import.py index 6b5ad73a..d961666c 100644 --- a/test/test_integration_import.py +++ b/test/test_integration_import.py @@ -18,17 +18,17 @@ def test_goto_definition_on_import(): def test_complete_on_empty_import(): assert Script("from datetime import").completions()[0].name == 'import' # should just list the files in the directory - assert 10 < len(Script("from .", path='').completions()) < 30 + assert 10 < len(Script("from .", path='whatever.py').completions()) < 30 # Global import - assert len(Script("from . import", 1, 5, '').completions()) > 30 + assert len(Script("from . import", 1, 5, 'whatever.py').completions()) > 30 # relative import - assert 10 < len(Script("from . import", 1, 6, '').completions()) < 30 + assert 10 < len(Script("from . import", 1, 6, 'whatever.py').completions()) < 30 # Global import - assert len(Script("from . import classes", 1, 5, '').completions()) > 30 + assert len(Script("from . import classes", 1, 5, 'whatever.py').completions()) > 30 # relative import - assert 10 < len(Script("from . import classes", 1, 6, '').completions()) < 30 + assert 10 < len(Script("from . import classes", 1, 6, 'whatever.py').completions()) < 30 wanted = set(['ImportError', 'import', 'ImportWarning']) assert set([c.name for c in Script("import").completions()]) == wanted diff --git a/test/test_new_parser.py b/test/test_new_parser.py index 8684fbd4..e66591dc 100644 --- a/test/test_new_parser.py +++ b/test/test_new_parser.py @@ -1,11 +1,11 @@ from jedi._compatibility import u -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar def test_basic_parsing(): def compare(string): """Generates the AST object and then regenerates the code.""" - assert Parser(load_grammar(), string).module.get_code() == string + assert ParserWithRecovery(load_grammar(), string).module.get_code() == string compare(u('\na #pass\n')) compare(u('wblabla* 1\t\n')) diff --git a/test/test_parser/test_fast_parser.py b/test/test_parser/test_fast_parser.py index 4d1f5f22..46cdf5c0 100644 --- a/test/test_parser/test_fast_parser.py +++ b/test/test_parser/test_fast_parser.py @@ -1,10 +1,13 @@ from textwrap import dedent +import pytest + import jedi from jedi._compatibility import u from jedi import cache from jedi.parser import load_grammar from jedi.parser.fast import FastParser +from jedi.parser.utils import save_parser def test_add_to_end(): @@ -38,7 +41,7 @@ def test_class_in_docstr(): Regression test for a problem with classes in docstrings. """ a = '"\nclasses\n"' - jedi.Script(a, 1, 0)._parser + jedi.Script(a, 1, 0)._get_module() b = a + '\nimport os' assert jedi.Script(b, 4, 8).goto_assignments() @@ -75,16 +78,17 @@ def test_split_parts(): test('a\n\n', 'def b(): pass\n', 'c\n') test('a\n', 'def b():\n pass\n', 'c\n') + test('from x\\\n') + test('a\n\\\n') + def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0): if number_of_splits is None: number_of_splits = number_parsers_used p = FastParser(load_grammar(), u(src)) - cache.save_parser(None, p, pickling=False) + save_parser(None, p, pickling=False) - # TODO Don't change get_code, the whole thing should be the same. - # -> Need to refactor the parser first, though. assert src == p.module.get_code() assert p.number_of_splits == number_of_splits assert p.number_parsers_used == number_parsers_used @@ -329,7 +333,7 @@ def test_wrong_indentation(): b a """) - check_fp(src, 1) + #check_fp(src, 1) src = dedent("""\ def complex(): @@ -346,13 +350,15 @@ def test_wrong_indentation(): def test_open_parentheses(): func = 'def func():\n a' - p = FastParser(load_grammar(), u('isinstance(\n\n' + func)) - # As you can see, the isinstance call cannot be seen anymore after - # get_code, because it isn't valid code. - assert p.module.get_code() == '\n\n' + func + code = u('isinstance(\n\n' + func) + p = FastParser(load_grammar(), code) + # As you can see, the part that was failing is still there in the get_code + # call. It is not relevant for evaluation, but still available as an + # ErrorNode. + assert p.module.get_code() == code assert p.number_of_splits == 2 assert p.number_parsers_used == 2 - cache.save_parser(None, p, pickling=False) + save_parser(None, p, pickling=False) # Now with a correct parser it should work perfectly well. check_fp('isinstance()\n' + func, 1, 2) @@ -420,6 +426,16 @@ def test_fake_parentheses(): check_fp(src, 3, 2, 1) +def test_additional_indent(): + source = dedent('''\ + int( + def x(): + pass + ''') + + check_fp(source, 2) + + def test_incomplete_function(): source = '''return ImportErr''' @@ -437,4 +453,44 @@ def test_string_literals(): """) script = jedi.Script(dedent(source)) + script._get_module().end_pos == (6, 0) assert script.completions() + + +def test_decorator_string_issue(): + """ + Test case from #589 + """ + source = dedent('''\ + """ + @""" + def bla(): + pass + + bla.''') + + s = jedi.Script(source) + assert s.completions() + assert s._get_module().get_code() == source + + +def test_round_trip(): + source = dedent(''' + def x(): + """hahaha""" + func''') + + f = FastParser(load_grammar(), u(source)) + assert f.get_parsed_node().get_code() == source + + +@pytest.mark.xfail() +def test_parentheses_in_string(): + code = dedent(''' + def x(): + '(' + + import abc + + abc.''') + check_fp(code, 2, 1, 1) diff --git a/test/test_parser/test_get_code.py b/test/test_parser/test_get_code.py index 8dc83aff..43202fb0 100644 --- a/test/test_parser/test_get_code.py +++ b/test/test_parser/test_get_code.py @@ -3,7 +3,7 @@ import difflib import pytest from jedi._compatibility import u -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar code_basic_features = u(''' """A mod docstring""" @@ -44,7 +44,7 @@ def diff_code_assert(a, b, n=4): def test_basic_parsing(): """Validate the parsing features""" - prs = Parser(load_grammar(), code_basic_features) + prs = ParserWithRecovery(load_grammar(), code_basic_features) diff_code_assert( code_basic_features, prs.module.get_code() @@ -53,7 +53,7 @@ def test_basic_parsing(): def test_operators(): src = u('5 * 3') - prs = Parser(load_grammar(), src) + prs = ParserWithRecovery(load_grammar(), src) diff_code_assert(src, prs.module.get_code()) @@ -82,7 +82,7 @@ def method_with_docstring(): """class docstr""" pass ''') - assert Parser(load_grammar(), s).module.get_code() == s + assert ParserWithRecovery(load_grammar(), s).module.get_code() == s def test_end_newlines(): @@ -92,7 +92,7 @@ def test_end_newlines(): line the parser needs. """ def test(source, end_pos): - module = Parser(load_grammar(), u(source)).module + module = ParserWithRecovery(load_grammar(), u(source)).module assert module.get_code() == source assert module.end_pos == end_pos @@ -103,3 +103,5 @@ def test_end_newlines(): test('a\n#comment', (2, 8)) test('a#comment', (1, 9)) test('def a():\n pass', (2, 5)) + + test('def a(', (1, 6)) diff --git a/test/test_parser/test_param_splitting.py b/test/test_parser/test_param_splitting.py new file mode 100644 index 00000000..f97c3dc5 --- /dev/null +++ b/test/test_parser/test_param_splitting.py @@ -0,0 +1,34 @@ +''' +To make the life of any analysis easier, we are generating Param objects +instead of simple parser objects. +''' + +from textwrap import dedent + +from jedi.parser import Parser, load_grammar + + +def assert_params(param_string, **wanted_dct): + source = dedent(''' + def x(%s): + pass + ''') % param_string + + parser = Parser(load_grammar(), dedent(source)) + funcdef = parser.get_parsed_node().subscopes[0] + dct = dict((p.name.value, p.default and p.default.get_code()) + for p in funcdef.params) + assert dct == wanted_dct + assert parser.get_parsed_node().get_code() == source + + +def test_split_params_with_separation_star(): + assert_params(u'x, y=1, *, z=3', x=None, y='1', z='3') + assert_params(u'*, x', x=None) + assert_params(u'*') + + +def test_split_params_with_stars(): + assert_params(u'x, *args', x=None, args=None) + assert_params(u'**kwargs', kwargs=None) + assert_params(u'*args, **kwargs', args=None, kwargs=None) diff --git a/test/test_parser/test_parser.py b/test/test_parser/test_parser.py index 31c92691..0ec75d0a 100644 --- a/test/test_parser/test_parser.py +++ b/test/test_parser/test_parser.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- import sys +from textwrap import dedent import jedi from jedi._compatibility import u, is_py3 -from jedi.parser import Parser, load_grammar -from jedi.parser.user_context import UserContextParser +from jedi.parser import ParserWithRecovery, load_grammar from jedi.parser import tree as pt -from textwrap import dedent def test_user_statement_on_import(): @@ -15,15 +14,16 @@ def test_user_statement_on_import(): " time)") for pos in [(2, 1), (2, 4)]: - p = UserContextParser(load_grammar(), s, None, pos, None, lambda x: 1).user_stmt() - assert isinstance(p, pt.Import) - assert [str(n) for n in p.get_defined_names()] == ['time'] + p = ParserWithRecovery(load_grammar(), s) + stmt = p.module.get_statement_for_position(pos) + assert isinstance(stmt, pt.Import) + assert [str(n) for n in stmt.get_defined_names()] == ['time'] class TestCallAndName(): def get_call(self, source): # Get the simple_stmt and then the first one. - simple_stmt = Parser(load_grammar(), u(source)).module.children[0] + simple_stmt = ParserWithRecovery(load_grammar(), u(source)).module.children[0] return simple_stmt.children[0] def test_name_and_call_positions(self): @@ -58,7 +58,7 @@ class TestCallAndName(): class TestSubscopes(): def get_sub(self, source): - return Parser(load_grammar(), u(source)).module.subscopes[0] + return ParserWithRecovery(load_grammar(), u(source)).module.subscopes[0] def test_subscope_names(self): name = self.get_sub('class Foo: pass').name @@ -74,7 +74,7 @@ class TestSubscopes(): class TestImports(): def get_import(self, source): - return Parser(load_grammar(), source).module.imports[0] + return ParserWithRecovery(load_grammar(), source).module.imports[0] def test_import_names(self): imp = self.get_import(u('import math\n')) @@ -89,13 +89,13 @@ class TestImports(): def test_module(): - module = Parser(load_grammar(), u('asdf'), 'example.py').module + module = ParserWithRecovery(load_grammar(), u('asdf'), 'example.py').module name = module.name assert str(name) == 'example' assert name.start_pos == (1, 0) assert name.end_pos == (1, 7) - module = Parser(load_grammar(), u('asdf')).module + module = ParserWithRecovery(load_grammar(), u('asdf')).module name = module.name assert str(name) == '' assert name.start_pos == (1, 0) @@ -108,7 +108,7 @@ def test_end_pos(): def func(): y = None ''')) - parser = Parser(load_grammar(), s) + parser = ParserWithRecovery(load_grammar(), s) scope = parser.module.subscopes[0] assert scope.start_pos == (3, 0) assert scope.end_pos == (5, 0) @@ -121,7 +121,7 @@ def test_carriage_return_statements(): # this is a namespace package ''')) source = source.replace('\n', '\r\n') - stmt = Parser(load_grammar(), source).module.statements[0] + stmt = ParserWithRecovery(load_grammar(), source).module.statements[0] assert '#' not in stmt.get_code() @@ -129,7 +129,7 @@ def test_incomplete_list_comprehension(): """ Shouldn't raise an error, same bug as #418. """ # With the old parser this actually returned a statement. With the new # parser only valid statements generate one. - assert Parser(load_grammar(), u('(1 for def')).module.statements == [] + assert ParserWithRecovery(load_grammar(), u('(1 for def')).module.statements == [] def test_hex_values_in_docstring(): @@ -141,7 +141,7 @@ def test_hex_values_in_docstring(): return 1 ''' - doc = Parser(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc + doc = ParserWithRecovery(load_grammar(), dedent(u(source))).module.subscopes[0].raw_doc if is_py3: assert doc == '\xff' else: @@ -160,9 +160,9 @@ def test_error_correction_with(): def test_newline_positions(): - endmarker = Parser(load_grammar(), u('a\n')).module.children[-1] + endmarker = ParserWithRecovery(load_grammar(), u('a\n')).module.children[-1] assert endmarker.end_pos == (2, 0) - new_line = endmarker.get_previous() + new_line = endmarker.get_previous_leaf() assert new_line.start_pos == (1, 1) assert new_line.end_pos == (2, 0) @@ -174,7 +174,7 @@ def test_end_pos_error_correction(): end_pos, even if something breaks in the parser (error correction). """ s = u('def x():\n .') - m = Parser(load_grammar(), s).module + m = ParserWithRecovery(load_grammar(), s).module func = m.children[0] assert func.type == 'funcdef' # This is not exactly correct, but ok, because it doesn't make a difference @@ -190,8 +190,8 @@ def test_param_splitting(): """ def check(src, result): # Python 2 tuple params should be ignored for now. - grammar = load_grammar('grammar%s.%s' % sys.version_info[:2]) - m = Parser(grammar, u(src)).module + grammar = load_grammar('%s.%s' % sys.version_info[:2]) + m = ParserWithRecovery(grammar, u(src)).module if is_py3: assert not m.subscopes else: @@ -211,5 +211,28 @@ def test_unicode_string(): def test_backslash_dos_style(): grammar = load_grammar() - m = Parser(grammar, u('\\\r\n')).module + m = ParserWithRecovery(grammar, u('\\\r\n')).module assert m + + +def test_started_lambda_stmt(): + p = ParserWithRecovery(load_grammar(), u'lambda a, b: a i') + assert p.get_parsed_node().children[0].type == 'error_node' + + +def test_python2_octal(): + parser = ParserWithRecovery(load_grammar(), u'0660') + first = parser.get_parsed_node().children[0] + if is_py3: + assert first.type == 'error_node' + else: + assert first.children[0].type == 'number' + + +def test_python3_octal(): + parser = ParserWithRecovery(load_grammar(), u'0o660') + module = parser.get_parsed_node() + if is_py3: + assert module.children[0].children[0].type == 'number' + else: + assert module.children[0].type == 'error_node' diff --git a/test/test_parser/test_parser_tree.py b/test/test_parser/test_parser_tree.py index 480230ba..57ea8259 100644 --- a/test/test_parser/test_parser_tree.py +++ b/test/test_parser/test_parser_tree.py @@ -5,17 +5,18 @@ from textwrap import dedent import pytest from jedi._compatibility import u, unicode -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar from jedi.parser import tree as pt class TestsFunctionAndLambdaParsing(object): FIXTURES = [ - ('def my_function(x, y, z):\n return x + y * z\n', { + ('def my_function(x, y, z) -> str:\n return x + y * z\n', { 'name': 'my_function', 'call_sig': 'my_function(x, y, z)', 'params': ['x', 'y', 'z'], + 'annotation': "str", }), ('lambda x, y, z: x + y * z\n', { 'name': '', @@ -26,7 +27,7 @@ class TestsFunctionAndLambdaParsing(object): @pytest.fixture(params=FIXTURES) def node(self, request): - parsed = Parser(load_grammar(), dedent(u(request.param[0]))) + parsed = ParserWithRecovery(load_grammar(), dedent(u(request.param[0]))) request.keywords['expected'] = request.param[1] return parsed.module.subscopes[0] @@ -55,7 +56,11 @@ class TestsFunctionAndLambdaParsing(object): assert not node.yields def test_annotation(self, node, expected): - assert node.annotation() is expected.get('annotation', None) + expected_annotation = expected.get('annotation', None) + if expected_annotation is None: + assert node.annotation() is None + else: + assert node.annotation().value == expected_annotation def test_get_call_signature(self, node, expected): assert node.get_call_signature() == expected['call_sig'] diff --git a/test/test_parser/test_pgen2.py b/test/test_parser/test_pgen2.py new file mode 100644 index 00000000..46f4938f --- /dev/null +++ b/test/test_parser/test_pgen2.py @@ -0,0 +1,279 @@ +"""Test suite for 2to3's parser and grammar files. + +This is the place to add tests for changes to 2to3's grammar, such as those +merging the grammars for Python 2 and 3. In addition to specific tests for +parts of the grammar we've changed, we also make sure we can parse the +test_grammar.py files from both Python 2 and Python 3. +""" + +from textwrap import dedent + + +from jedi._compatibility import unicode, is_py3 +from jedi.parser import Parser, load_grammar, ParseError +import pytest + +from test.helpers import TestCase + + +def parse(code, version='3.4'): + code = dedent(code) + "\n\n" + grammar = load_grammar(version=version) + return Parser(grammar, unicode(code), 'file_input').get_parsed_node() + + +class TestDriver(TestCase): + + def test_formfeed(self): + s = """print 1\n\x0Cprint 2\n""" + t = parse(s, '2.7') + self.assertEqual(t.children[0].children[0].type, 'print_stmt') + self.assertEqual(t.children[1].children[0].type, 'print_stmt') + s = """1\n\x0C\x0C2\n""" + t = parse(s, '2.7') + + +class GrammarTest(TestCase): + def invalid_syntax(self, code, **kwargs): + try: + parse(code, **kwargs) + except ParseError: + pass + else: + raise AssertionError("Syntax shouldn't have been valid") + + +class TestMatrixMultiplication(GrammarTest): + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + def test_matrix_multiplication_operator(self): + parse("a @ b", "3.5") + parse("a @= b", "3.5") + + +class TestYieldFrom(GrammarTest): + def test_yield_from(self): + parse("yield from x") + parse("(yield from x) + y") + self.invalid_syntax("yield from") + + +class TestAsyncAwait(GrammarTest): + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + def test_await_expr(self): + parse("""async def foo(): + await x + """, "3.5") + + parse("""async def foo(): + + def foo(): pass + + def foo(): pass + + await x + """, "3.5") + + parse("""async def foo(): return await a""", "3.5") + + parse("""def foo(): + def foo(): pass + async def foo(): await x + """, "3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_await_expr_invalid(self): + self.invalid_syntax("await x", version="3.5") + self.invalid_syntax("""def foo(): + await x""", version="3.5") + + self.invalid_syntax("""def foo(): + def foo(): pass + async def foo(): pass + await x + """, version="3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_async_var(self): + parse("""async = 1""", "3.5") + parse("""await = 1""", "3.5") + parse("""def async(): pass""", "3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + def test_async_for(self): + parse("""async def foo(): + async for a in b: pass""", "3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_async_for_invalid(self): + self.invalid_syntax("""def foo(): + async for a in b: pass""", version="3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + def test_async_with(self): + parse("""async def foo(): + async with a: pass""", "3.5") + + @pytest.mark.skipif('sys.version_info[:2] < (3, 5)') + @pytest.mark.xfail(reason="acting like python 3.7") + def test_async_with_invalid(self): + self.invalid_syntax("""def foo(): + async with a: pass""", version="3.5") + + +class TestRaiseChanges(GrammarTest): + def test_2x_style_1(self): + parse("raise") + + def test_2x_style_2(self): + parse("raise E, V", version='2.7') + + def test_2x_style_3(self): + parse("raise E, V, T", version='2.7') + + def test_2x_style_invalid_1(self): + self.invalid_syntax("raise E, V, T, Z", version='2.7') + + def test_3x_style(self): + parse("raise E1 from E2") + + def test_3x_style_invalid_1(self): + self.invalid_syntax("raise E, V from E1") + + def test_3x_style_invalid_2(self): + self.invalid_syntax("raise E from E1, E2") + + def test_3x_style_invalid_3(self): + self.invalid_syntax("raise from E1, E2") + + def test_3x_style_invalid_4(self): + self.invalid_syntax("raise E from") + + +# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef +class TestFunctionAnnotations(GrammarTest): + def test_1(self): + parse("""def f(x) -> list: pass""") + + def test_2(self): + parse("""def f(x:int): pass""") + + def test_3(self): + parse("""def f(*x:str): pass""") + + def test_4(self): + parse("""def f(**x:float): pass""") + + def test_5(self): + parse("""def f(x, y:1+2): pass""") + + def test_6(self): + self.invalid_syntax("""def f(a, (b:1, c:2, d)): pass""") + + def test_7(self): + self.invalid_syntax("""def f(a, (b:1, c:2, d), e:3=4, f=5, *g:6): pass""") + + def test_8(self): + s = """def f(a, (b:1, c:2, d), e:3=4, f=5, + *g:6, h:7, i=8, j:9=10, **k:11) -> 12: pass""" + self.invalid_syntax(s) + + +class TestExcept(GrammarTest): + def test_new(self): + s = """ + try: + x + except E as N: + y""" + parse(s) + + def test_old(self): + s = """ + try: + x + except E, N: + y""" + parse(s, version='2.7') + + +# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testAtoms +class TestSetLiteral(GrammarTest): + def test_1(self): + parse("""x = {'one'}""") + + def test_2(self): + parse("""x = {'one', 1,}""") + + def test_3(self): + parse("""x = {'one', 'two', 'three'}""") + + def test_4(self): + parse("""x = {2, 3, 4,}""") + + +class TestNumericLiterals(GrammarTest): + def test_new_octal_notation(self): + code = """0o7777777777777""" + if is_py3: + parse(code) + else: + self.invalid_syntax(code) + self.invalid_syntax("""0o7324528887""") + + def test_new_binary_notation(self): + parse("""0b101010""") + self.invalid_syntax("""0b0101021""") + + +class TestClassDef(GrammarTest): + def test_new_syntax(self): + parse("class B(t=7): pass") + parse("class B(t, *args): pass") + parse("class B(t, **kwargs): pass") + parse("class B(t, *args, **kwargs): pass") + parse("class B(t, y=9, *args, **kwargs): pass") + + +class TestParserIdempotency(TestCase): + """A cut-down version of pytree_idempotency.py.""" + def test_extended_unpacking(self): + parse("a, *b, c = x\n") + parse("[*a, b] = x\n") + parse("(z, *y, w) = m\n") + parse("for *z, m in d: pass\n") + + +class TestLiterals(GrammarTest): + # It's not possible to get the same result when using \xaa in Python 2/3, + # because it's treated differently. + @pytest.mark.skipif('sys.version_info[0] < 3') + def test_multiline_bytes_literals(self): + s = """ + md5test(b"\xaa" * 80, + (b"Test Using Larger Than Block-Size Key " + b"and Larger Than One Block-Size Data"), + "6f630fad67cda0ee1fb1f562db3aa53e") + """ + parse(s) + + def test_multiline_bytes_tripquote_literals(self): + s = ''' + b""" + + + """ + ''' + parse(s) + + @pytest.mark.skipif('sys.version_info[0] < 3') + def test_multiline_str_literals(self): + s = """ + md5test("\xaa" * 80, + ("Test Using Larger Than Block-Size Key " + "and Larger Than One Block-Size Data"), + "6f630fad67cda0ee1fb1f562db3aa53e") + """ + parse(s) diff --git a/test/test_parser/test_tokenize.py b/test/test_parser/test_tokenize.py index f038cc8d..e53f85a6 100644 --- a/test/test_parser/test_tokenize.py +++ b/test/test_parser/test_tokenize.py @@ -7,7 +7,7 @@ import pytest from jedi._compatibility import u, is_py3 from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT -from jedi.parser import Parser, load_grammar, tokenize +from jedi.parser import ParserWithRecovery, load_grammar, tokenize from ..helpers import unittest @@ -15,7 +15,7 @@ from ..helpers import unittest class TokenTest(unittest.TestCase): def test_end_pos_one_line(self): - parsed = Parser(load_grammar(), dedent(u(''' + parsed = ParserWithRecovery(load_grammar(), dedent(u(''' def testit(): a = "huhu" '''))) @@ -23,7 +23,7 @@ class TokenTest(unittest.TestCase): assert tok.end_pos == (3, 14) def test_end_pos_multi_line(self): - parsed = Parser(load_grammar(), dedent(u(''' + parsed = ParserWithRecovery(load_grammar(), dedent(u(''' def testit(): a = """huhu asdfasdf""" + "h" @@ -108,7 +108,7 @@ class TokenTest(unittest.TestCase): ] for s in string_tokens: - parsed = Parser(load_grammar(), u('''a = %s\n''' % s)) + parsed = ParserWithRecovery(load_grammar(), u('''a = %s\n''' % s)) simple_stmt = parsed.module.children[0] expr_stmt = simple_stmt.children[0] assert len(expr_stmt.children) == 3 diff --git a/test/test_regression.py b/test/test_regression.py index 9e673154..5452349e 100644 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -15,7 +15,7 @@ from jedi._compatibility import u from jedi import Script from jedi import api from jedi.evaluate import imports -from jedi.parser import Parser, load_grammar +from jedi.parser import ParserWithRecovery, load_grammar #jedi.set_debug_function() @@ -102,7 +102,7 @@ class TestRegression(TestCase): def test_end_pos_line(self): # jedi issue #150 s = u("x()\nx( )\nx( )\nx ( )") - parser = Parser(load_grammar(), s) + parser = ParserWithRecovery(load_grammar(), s) for i, s in enumerate(parser.module.statements): assert s.end_pos == (i + 1, i + 3) @@ -146,7 +146,7 @@ class TestRegression(TestCase): x = 0 a = \ [1, 2, 3, 4, 5, 6, 7, 8, 9, (x)] # <-- here - """, '(x)] # <-- here', []) + """, '(x)] # <-- here', ['int']) def test_generator(self): # Did have some problems with the usage of generator completions this diff --git a/test/test_utils.py b/test/test_utils.py index 33353de1..2eb6e8ca 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -34,7 +34,7 @@ class TestSetupReadline(unittest.TestCase): def test_simple(self): assert self.completions('list') == ['list'] assert self.completions('importerror') == ['ImportError'] - s = "print BaseE" + s = "print(BaseE" assert self.completions(s) == [s + 'xception'] def test_nested(self): @@ -53,7 +53,8 @@ class TestSetupReadline(unittest.TestCase): try: assert self.completions('os.path.join') == ['os.path.join'] - assert self.completions('os.path.join().upper') == ['os.path.join().upper'] + string = 'os.path.join("a").upper' + assert self.completions(string) == [string] c = set(['os.' + d for d in dir(os) if d.startswith('ch')]) assert set(self.completions('os.ch')) == set(c) diff --git a/tox.ini b/tox.ini index 526092fc..dadbb2fe 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py32, py33, py34 +envlist = py26, py27, py33, py34, py35 [testenv] deps = pytest>=2.3.5 @@ -18,6 +18,22 @@ commands = deps = unittest2 {[testenv]deps} +[testenv:py27] +deps = +# for testing the typing module + typing + {[testenv]deps} +[testenv:py33] +deps = + typing + {[testenv]deps} +[testenv:py34] +deps = + typing + {[testenv]deps} +[testenv:py35] +deps = + {[testenv]deps} [testenv:cov] deps = coverage