From a31ba8737abff01afbf169d3fb181dd2498ee5ad Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 06:46:31 +0100 Subject: [PATCH 001/109] Run refactoring test using py.test refactor.collect_file_tests is fixed; it uses global variable refactoring_test_dir which is not defined when refactor is used as a module. --- test/conftest.py | 11 ++++++++--- test/refactor.py | 7 +++---- test/test_integration.py | 11 +++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 3ed26cfb..ae57594f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,6 +2,7 @@ from os.path import join, dirname, abspath default_base_dir = join(dirname(abspath(__file__)), 'completion') import run +import refactor def pytest_addoption(parser): @@ -38,11 +39,15 @@ def pytest_generate_tests(metafunc): """ :type metafunc: _pytest.python.Metafunc """ + base_dir = metafunc.config.option.base_dir + test_files = dict(map(parse_test_files_option, + metafunc.config.option.test_files)) if 'case' in metafunc.fixturenames: - base_dir = metafunc.config.option.base_dir - test_files = dict(map(parse_test_files_option, - metafunc.config.option.test_files)) thirdparty = metafunc.config.option.thirdparty metafunc.parametrize( 'case', run.collect_dir_tests(base_dir, test_files, thirdparty)) + if 'refactor_case' in metafunc.fixturenames: + metafunc.parametrize( + 'refactor_case', + refactor.collect_dir_tests(base_dir, test_files)) diff --git a/test/refactor.py b/test/refactor.py index e4cc7c08..83bcaa98 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -64,7 +64,7 @@ class RefactoringCase(object): self.name, self.line_nr - 1) -def collect_file_tests(source, f_name, lines_to_execute): +def collect_file_tests(source, path, lines_to_execute): r = r'^# --- ?([^\n]*)\n((?:(?!\n# \+\+\+).)*)' \ r'\n# \+\+\+((?:(?!\n# ---).)*)' for match in re.finditer(r, source, re.DOTALL | re.MULTILINE): @@ -86,7 +86,6 @@ def collect_file_tests(source, f_name, lines_to_execute): if lines_to_execute and line_nr - 1 not in lines_to_execute: continue - path = os.path.join(os.path.abspath(refactoring_test_dir), f_name) yield RefactoringCase(name, source, line_nr, index, path, new_name, start_line_test, second) @@ -96,10 +95,10 @@ def collect_dir_tests(base_dir, test_files): files_to_execute = [a for a in test_files.items() if a[0] in f_name] lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, []) if f_name.endswith(".py") and (not test_files or files_to_execute): - path = os.path.join(refactoring_test_dir, f_name) + path = os.path.join(base_dir, f_name) with open(path) as f: source = f.read() - for case in collect_file_tests(source, f_name, lines_to_execute): + for case in collect_file_tests(source, path, lines_to_execute): yield case diff --git a/test/test_integration.py b/test/test_integration.py index 69ef90e1..c481fb5c 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -101,3 +101,14 @@ def test_integration(case, monkeypatch, pytestconfig): TEST_USAGES: run_related_name_test, } testers[case.test_type](case) + + +def test_refactor(refactor_case): + """ + Run refactoring test case. + + :type refactor_case: :class:`.refactor.RefactoringCase` + """ + refactor_case.run() + result, desired = refactor_case.result, refactor_case.desired + assert result == desired, "Refactoring test %r fails" % refactor_case From a993dd0da44cb4425be58bcf8b6b2c7eeb9212c7 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:00:44 +0100 Subject: [PATCH 002/109] Fix test_refactor It was not run because test cases were collected from test/complete instead of test/refactor. --- test/conftest.py | 14 ++++++++++---- test/test_integration.py | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index ae57594f..21f2e0e4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,14 +1,19 @@ -from os.path import join, dirname, abspath -default_base_dir = join(dirname(abspath(__file__)), 'completion') +import os +import base import run import refactor def pytest_addoption(parser): parser.addoption( - "--base-dir", default=default_base_dir, + "--integration-case-dir", + default=os.path.join(base.test_dir, 'completion'), help="Directory in which integration test case files locate.") + parser.addoption( + "--refactor-case-dir", + default=os.path.join(base.test_dir, 'refactor'), + help="Directory in which refactoring test case files locate.") parser.addoption( "--test-files", "-T", default=[], action='append', help=( @@ -39,15 +44,16 @@ def pytest_generate_tests(metafunc): """ :type metafunc: _pytest.python.Metafunc """ - base_dir = metafunc.config.option.base_dir test_files = dict(map(parse_test_files_option, metafunc.config.option.test_files)) if 'case' in metafunc.fixturenames: + base_dir = metafunc.config.option.integration_case_dir thirdparty = metafunc.config.option.thirdparty metafunc.parametrize( 'case', run.collect_dir_tests(base_dir, test_files, thirdparty)) if 'refactor_case' in metafunc.fixturenames: + base_dir = metafunc.config.option.refactor_case_dir metafunc.parametrize( 'refactor_case', refactor.collect_dir_tests(base_dir, test_files)) diff --git a/test/test_integration.py b/test/test_integration.py index c481fb5c..fe09013a 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,6 +1,7 @@ import os import re +import base from run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES @@ -92,7 +93,7 @@ def run_related_name_test(case): def test_integration(case, monkeypatch, pytestconfig): - repo_root = os.path.dirname(os.path.dirname(pytestconfig.option.base_dir)) + repo_root = base.root_dir monkeypatch.chdir(os.path.join(repo_root, 'jedi')) testers = { TEST_COMPLETIONS: run_completion_test, From 51a094be028d4275d1265792158a907506ccdecf Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:05:30 +0100 Subject: [PATCH 003/109] Run py.test in tox.ini --- pytest.ini | 2 ++ tox.ini | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..2ce9da86 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --assert=plain diff --git a/tox.ini b/tox.ini index 2a39e11c..afcc4cbf 100644 --- a/tox.ini +++ b/tox.ini @@ -4,13 +4,12 @@ envlist = py25, py26, py27, py32 setenv = XDG_CACHE_HOME={envtmpdir}/cache deps = - nose + pytest commands = - python regression.py - python run.py - python refactor.py - nosetests --with-doctest --doctest-tests {toxinidir}/jedi -changedir = test + py.test [] + # Doctests can't be run with the main tests because then py.test + # tries to import broken python files under test/*/. + py.test --doctest-modules {toxinidir}/jedi [testenv:py25] deps = simplejson From 0f9761aac73d3c6de849b83280944437d7965d25 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:11:37 +0100 Subject: [PATCH 004/109] Fix tests for Python 3.2 (use relative import) --- test/conftest.py | 6 +++--- test/refactor.py | 2 +- test/regression.py | 2 +- test/run.py | 2 +- test/test_integration.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 21f2e0e4..8d94cbd3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,8 +1,8 @@ import os -import base -import run -import refactor +from . import base +from . import run +from . import refactor def pytest_addoption(parser): diff --git a/test/refactor.py b/test/refactor.py index 83bcaa98..058af2a2 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -10,7 +10,7 @@ import traceback import re import itertools -import base +from . import base from jedi._compatibility import reduce import jedi diff --git a/test/regression.py b/test/regression.py index 54dfe755..b327811c 100755 --- a/test/regression.py +++ b/test/regression.py @@ -11,7 +11,7 @@ import itertools import os import textwrap -from base import TestBase, unittest, cwd_at +from .base import TestBase, unittest, cwd_at import jedi from jedi._compatibility import is_py25, utf8, unicode diff --git a/test/run.py b/test/run.py index 106f6359..2692439f 100755 --- a/test/run.py +++ b/test/run.py @@ -54,7 +54,7 @@ import re import traceback import itertools -import base +from . import base from jedi._compatibility import unicode, StringIO, reduce, literal_eval, is_py25 diff --git a/test/test_integration.py b/test/test_integration.py index fe09013a..0e408c4f 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,8 +1,8 @@ import os import re -import base -from run import \ +from . import base +from .run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES import jedi From 180d0a87648959cb082d51ad2234526048374f1a Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:12:44 +0100 Subject: [PATCH 005/109] Rename regression.py to test_regression.py in order to let py.test collect the tests. --- test/{regression.py => test_regression.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{regression.py => test_regression.py} (100%) diff --git a/test/regression.py b/test/test_regression.py similarity index 100% rename from test/regression.py rename to test/test_regression.py From 71bb93224d82b1eabebe08076144266e0afd321c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:48:20 +0100 Subject: [PATCH 006/109] Ignore first N failures in Python 2.5 --- test/base.py | 32 ++++++++++++++++++++++++++++++++ test/test_integration.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/test/base.py b/test/base.py index a9e2d9ab..4277bb71 100644 --- a/test/base.py +++ b/test/base.py @@ -12,8 +12,11 @@ test_dir = dirname(abspath(__file__)) root_dir = dirname(test_dir) sys.path.insert(0, root_dir) +import pytest + import jedi from jedi import debug +from jedi._compatibility import is_py25 test_sum = 0 t_start = time.time() @@ -102,3 +105,32 @@ def cwd_at(path): os.chdir(oldcwd) return wrapper return decorator + + +_py25_fails = 0 +py25_allowed_fails = 9 + + +def skip_py25_fails(func): + """ + Skip first `py25_allowed_fails` failures in Python 2.5. + + .. todo:: Remove this decorator by implementing "skip tag" for + integration tests. + """ + @functools.wraps(func) + def wrapper(*args, **kwds): + global _py25_fails + try: + func(*args, **kwds) + except AssertionError: + _py25_fails += 1 + if _py25_fails > py25_allowed_fails: + raise + else: + pytest.skip("%d-th failure (there can be %d failures)" % + (_py25_fails, py25_allowed_fails)) + return wrapper + +if not is_py25: + skip_py25_fails = lambda f: f diff --git a/test/test_integration.py b/test/test_integration.py index 0e408c4f..f693b31b 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -101,7 +101,7 @@ def test_integration(case, monkeypatch, pytestconfig): TEST_ASSIGNMENTS: run_goto_test, TEST_USAGES: run_related_name_test, } - testers[case.test_type](case) + base.skip_py25_fails(testers[case.test_type])(case) def test_refactor(refactor_case): From 13b48632e599ab5e56f394e78c0f81a608af3af9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:58:40 +0100 Subject: [PATCH 007/109] Better assertion message formatter --- test/test_integration.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/test/test_integration.py b/test/test_integration.py index f693b31b..f348689d 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -9,16 +9,28 @@ import jedi from jedi._compatibility import literal_eval +def assert_case_equal(case, actual, desired): + """ + Assert ``actual == desired`` with formatted message. + + This is not needed for typical py.test use case, but as we need + ``--assert=plain`` (see ../pytest.ini) to workaround some issue + due to py.test magic, let's format the message by hand. + """ + assert actual == desired, """ +Test %r failed. +actual = %s +desired = %s +""" % (case, actual, desired) + + def run_completion_test(case): (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) completions = script.complete() #import cProfile; cProfile.run('script.complete()') comp_str = set([c.word for c in completions]) - if comp_str != set(literal_eval(correct)): - raise AssertionError( - 'Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, comp_str, correct)) + assert_case_equal(case, comp_str, set(literal_eval(correct))) def run_definition_test(case): @@ -53,19 +65,14 @@ def run_definition_test(case): should_str = definition(correct, start, script.source_path) result = script.definition() is_str = set(r.desc_with_module for r in result) - if is_str != should_str: - raise AssertionError( - 'Solution @%s not right, received %s, wanted %s' - % (line_nr - 1, is_str, should_str)) + assert_case_equal(case, is_str, should_str) def run_goto_test(case): (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) result = script.goto() comp_str = str(sorted(str(r.description) for r in result)) - if comp_str != correct: - raise AssertionError('Solution @%s not right, received %s, wanted %s' - % (line_nr - 1, comp_str, correct)) + assert_case_equal(case, comp_str, correct) def run_related_name_test(case): @@ -86,10 +93,7 @@ def run_related_name_test(case): else: wanted.append(('renaming', line_nr + pos_tup[0], pos_tup[1])) - wanted = sorted(wanted) - if compare != wanted: - raise AssertionError('Solution @%s not right, received %s, wanted %s' - % (line_nr - 1, compare, wanted)) + assert_case_equal(case, compare, sorted(wanted)) def test_integration(case, monkeypatch, pytestconfig): @@ -111,5 +115,5 @@ def test_refactor(refactor_case): :type refactor_case: :class:`.refactor.RefactoringCase` """ refactor_case.run() - result, desired = refactor_case.result, refactor_case.desired - assert result == desired, "Refactoring test %r fails" % refactor_case + assert_case_equal(refactor_case, + refactor_case.result, refactor_case.desired) From 88adcbcf8ac77f0c90b0a90147a06264ffbe2151 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 08:03:55 +0100 Subject: [PATCH 008/109] Use tox in .travis.yml --- .travis.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba8fb069..81b0cea4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,13 +5,9 @@ python: - 2.7 - 3.2 install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - pip install --use-mirrors simplejson unittest2; - fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then - pip install --use-mirrors unittest2; - fi - - pip install --use-mirrors nose + - pip install --use-mirrors tox script: - - cd test - - ./test.sh + - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | + sed --regexp-extended 's/([0-9])\.([0-9])/py\1\2/g') + - echo "TOXENV=$TOXENV" + - tox From 674743b7bbb71a2bf4003dd38daa224fa8dcaf56 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 08:49:24 +0100 Subject: [PATCH 009/109] Avoid NoSSLError in Python 2.5 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index afcc4cbf..bc94b159 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ commands = deps = simplejson unittest2 + ssl {[testenv]deps} [testenv:py26] deps = From e7b352b82617f392728e4fd4502311bfedefbeab Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:01:56 +0100 Subject: [PATCH 010/109] Install ssl in .travis.yml --- .travis.yml | 3 +++ tox.ini | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 81b0cea4..3db7151f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ python: - 2.7 - 3.2 install: + - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then + pip install --use-mirrors ssl; + fi - pip install --use-mirrors tox script: - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | diff --git a/tox.ini b/tox.ini index bc94b159..afcc4cbf 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,6 @@ commands = deps = simplejson unittest2 - ssl {[testenv]deps} [testenv:py26] deps = From 9e600ed0b15bd82b9ffa30c1c1e64dd14dbd56ec Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:04:56 +0100 Subject: [PATCH 011/109] Use --insecure when running pip with Python 2.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3db7151f..77b753ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - 3.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - pip install --use-mirrors ssl; + pip install --use-mirrors --insecure ssl; fi - pip install --use-mirrors tox script: From 135dd56e61bd97f4c30197bd9c49c4fcbc0bf0d8 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:15:30 +0100 Subject: [PATCH 012/109] Install libssl-dev for Python 2.5 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 77b753ea..01e5be0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,8 @@ python: - 3.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - pip install --use-mirrors --insecure ssl; + sudo apt-get install libssl-dev && + pip install --use-mirrors ssl; fi - pip install --use-mirrors tox script: From ab33400f76486d389319ba3f1385973002ea11f6 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:18:40 +0100 Subject: [PATCH 013/109] Install libbluetooth-dev for Python 2.5 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 01e5be0b..294f1503 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,9 @@ python: - 2.7 - 3.2 install: + # See: http://stackoverflow.com/questions/3241658/ - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - sudo apt-get install libssl-dev && + sudo apt-get install libbluetooth-dev && pip install --use-mirrors ssl; fi - pip install --use-mirrors tox From 5c3252908fc42a2887e3dcdda339d9ca260f6f56 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:29:00 +0100 Subject: [PATCH 014/109] Use PIP_INSECURE=t --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 294f1503..7a5adc19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,8 @@ python: - 2.7 - 3.2 install: - # See: http://stackoverflow.com/questions/3241658/ - - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - sudo apt-get install libbluetooth-dev && - pip install --use-mirrors ssl; - fi + # Needed only for Python 2.5: + - export PIP_INSECURE=t - pip install --use-mirrors tox script: - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | From cee167e3d29c951e51894291eb65ec700082a874 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 08:50:32 +0100 Subject: [PATCH 015/109] Run py.test in clean cache directory And finally remove XDG_CACHE_HOME=... in tox.ini. --- pytest.ini | 5 +++++ test/conftest.py | 26 ++++++++++++++++++++++++++ test/test_integration.py | 2 ++ tox.ini | 2 -- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 2ce9da86..4afbf164 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,7 @@ [pytest] addopts = --assert=plain + +# Activate `clean_jedi_cache` fixture for all tests. This should be +# fine as long as we are using `clean_jedi_cache` as a session scoped +# fixture. +usefixtures = clean_jedi_cache diff --git a/test/conftest.py b/test/conftest.py index 8d94cbd3..4cb2f4e5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,8 @@ import os +import shutil +import tempfile + +import pytest from . import base from . import run @@ -57,3 +61,25 @@ def pytest_generate_tests(metafunc): metafunc.parametrize( 'refactor_case', refactor.collect_dir_tests(base_dir, test_files)) + + +@pytest.fixture(scope='session') +def clean_jedi_cache(request): + """ + Set `jedi.settings.cache_directory` to a temporary directory during test. + + Note that you can't use built-in `tmpdir` and `monkeypatch` + fixture here because their scope is 'function', which is not used + in 'session' scope fixture. + + This fixture is activated in ../pytest.ini. + """ + settings = base.jedi.settings + old = settings.cache_directory + tmp = tempfile.mkdtemp(prefix='jedi-test-') + settings.cache_directory = tmp + + @request.addfinalizer + def restore(): + settings.cache_directory = old + shutil.rmtree(tmp) diff --git a/test/test_integration.py b/test/test_integration.py index f348689d..b3baf336 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,6 +1,8 @@ import os import re +import pytest + from . import base from .run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES diff --git a/tox.ini b/tox.ini index afcc4cbf..55022b23 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,6 @@ [tox] envlist = py25, py26, py27, py32 [testenv] -setenv = - XDG_CACHE_HOME={envtmpdir}/cache deps = pytest commands = From 7c289ce6be981eaf02f71dd11135a3a30e765235 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:52:58 +0100 Subject: [PATCH 016/109] Workaround test failure due to cache in Python 3.2 --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 55022b23..9ab832ab 100644 --- a/tox.ini +++ b/tox.ini @@ -17,3 +17,8 @@ deps = deps = unittest2 {[testenv]deps} +[testenv:py32] +# TODO: Without this setting, test uses ~/.cache/jedi/. +# There could be a bug due to import hack. +setenv = + XDG_CACHE_HOME={envtmpdir}/cache From 49f635dca3f8e8e6978f4a056c90184174f46257 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:01:57 +0100 Subject: [PATCH 017/109] Add a failing test due to import hack --- test/test_regression.py | 13 +++++++++++++ tox.ini | 1 + 2 files changed, 14 insertions(+) diff --git a/test/test_regression.py b/test/test_regression.py index b327811c..2e0da3cf 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -11,6 +11,8 @@ import itertools import os import textwrap +import pytest + from .base import TestBase, unittest, cwd_at import jedi @@ -524,5 +526,16 @@ class TestSpeed(TestBase): script.function_definition() #print(jedi.imports.imports_processed) + +@pytest.mark.skipif("sys.version_info >= (3,0)") +def test_settings_module(): + """ + jedi.settings and jedi.cache.settings must be the same module. + """ + from jedi import cache + from jedi import settings + assert cache.settings is settings + + if __name__ == '__main__': unittest.main() diff --git a/tox.ini b/tox.ini index 9ab832ab..25014940 100644 --- a/tox.ini +++ b/tox.ini @@ -20,5 +20,6 @@ deps = [testenv:py32] # TODO: Without this setting, test uses ~/.cache/jedi/. # There could be a bug due to import hack. +# See test_settings_module in test/test_regression.py. setenv = XDG_CACHE_HOME={envtmpdir}/cache From ff80988a75b6f1bcf6ecfff5ef837d2e02e8ef3c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:39:58 +0100 Subject: [PATCH 018/109] Remove old test code --- test/refactor.py | 60 +---------- test/run.py | 269 ++++++----------------------------------------- test/test.sh | 8 -- 3 files changed, 35 insertions(+), 302 deletions(-) delete mode 100755 test/test.sh diff --git a/test/refactor.py b/test/refactor.py index 058af2a2..206e1662 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -10,8 +10,7 @@ import traceback import re import itertools -from . import base - +from . import base # required to setup import path from jedi._compatibility import reduce import jedi from jedi import refactoring @@ -100,60 +99,3 @@ def collect_dir_tests(base_dir, test_files): source = f.read() for case in collect_file_tests(source, path, lines_to_execute): yield case - - -def run_test(cases): - """ - This is the completion test for some cases. The tests are not unit test - like, they are rather integration tests. - It uses comments to specify a test in the next line. The comment also says, - which results are expected. The comment always begins with `#?`. The last - row symbolizes the cursor. - - For example:: - - #? ['ab'] - ab = 3; a - - #? int() - ab = 3; ab - """ - fails = 0 - tests = 0 - for case in cases: - try: - if not case.check(): - print(case) - print(' ' + repr(str(case.result))) - print(' ' + repr(case.desired)) - fails += 1 - except Exception: - print(traceback.format_exc()) - print(case) - fails += 1 - tests += 1 - return tests, fails - - -def test_dir(refactoring_test_dir): - for (path, cases) in itertools.groupby( - collect_dir_tests(refactoring_test_dir, test_files), - lambda case: case.path): - num_tests, fails = run_test(cases) - - base.test_sum += num_tests - f_name = os.path.basename(path) - s = 'run %s tests with %s fails (%s)' % (num_tests, fails, f_name) - base.tests_fail += fails - print(s) - base.summary.append(s) - - -if __name__ == '__main__': - refactoring_test_dir = os.path.join(base.test_dir, 'refactor') - test_files = base.get_test_list() - test_dir(refactoring_test_dir) - - base.print_summary() - - sys.exit(1 if base.tests_fail else 0) diff --git a/test/run.py b/test/run.py index 2692439f..10cdf26d 100755 --- a/test/run.py +++ b/test/run.py @@ -31,35 +31,58 @@ If you want to debug a test, just use the --debug option. Auto-Completion +++++++++++++++ -.. autofunction:: run_completion_test +Uses comments to specify a test in the next line. The comment says, which +results are expected. The comment always begins with `#?`. The last row +symbolizes the cursor. + +For example:: + + #? ['real'] + a = 3; a.rea + +Because it follows ``a.rea`` and a is an ``int``, which has a ``real`` +property. Definition ++++++++++ -.. autofunction:: run_definition_test +Definition tests use the same symbols like completion tests. This is +possible because the completion tests are defined with a list:: + + #? int() + ab = 3; ab Goto ++++ -.. autofunction:: run_goto_test +Tests look like this:: + + abc = 1 + #! ['abc=1'] + abc + +Additionally it is possible to add a number which describes to position of +the test (otherwise it's just end of line):: + + #! 2 ['abc=1'] + abc Related Names +++++++++++++ -.. autofunction:: run_related_name_test +Tests look like this:: + + abc = 1 + #< abc@1,0 abc@3,0 + abc """ import os import sys import re -import traceback -import itertools - -from . import base - -from jedi._compatibility import unicode, StringIO, reduce, literal_eval, is_py25 +from . import base # required to setup import path import jedi -from jedi import debug +from jedi._compatibility import unicode, StringIO, reduce, is_py25 sys.path.pop(0) # pop again, because it might affect the completion @@ -71,147 +94,6 @@ TEST_ASSIGNMENTS = 2 TEST_USAGES = 3 -def run_completion_test(case): - """ - Uses comments to specify a test in the next line. The comment says, which - results are expected. The comment always begins with `#?`. The last row - symbolizes the cursor. - - For example:: - - #? ['real'] - a = 3; a.rea - - Because it follows ``a.rea`` and a is an ``int``, which has a ``real`` - property. - - Returns 1 for fail and 0 for success. - """ - (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) - completions = script.complete() - #import cProfile; cProfile.run('script.complete()') - - comp_str = set([c.word for c in completions]) - if comp_str != set(literal_eval(correct)): - print('Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, comp_str, correct)) - return 1 - return 0 - - -def run_definition_test(case): - """ - Definition tests use the same symbols like completion tests. This is - possible because the completion tests are defined with a list:: - - #? int() - ab = 3; ab - - Returns 1 for fail and 0 for success. - """ - def definition(correct, correct_start, path): - def defs(line_nr, indent): - s = jedi.Script(script.source, line_nr, indent, path) - return set(s.definition()) - - should_be = set() - number = 0 - for index in re.finditer('(?: +|$)', correct): - if correct == ' ': - continue - # -1 for the comment, +3 because of the comment start `#? ` - start = index.start() - if base.print_debug: - jedi.set_debug_function(None) - number += 1 - try: - should_be |= defs(line_nr - 1, start + correct_start) - except Exception: - print('could not resolve %s indent %s' % (line_nr - 1, start)) - raise - if base.print_debug: - jedi.set_debug_function(debug.print_to_stdout) - # because the objects have different ids, `repr` it, then compare it. - should_str = set(r.desc_with_module for r in should_be) - if len(should_str) < number: - raise Exception('Solution @%s not right, too few test results: %s' - % (line_nr - 1, should_str)) - return should_str - - (correct, line_nr, column, start, line) = \ - (case.correct, case.line_nr, case.column, case.start, case.line) - script = case.script() - should_str = definition(correct, start, script.source_path) - result = script.definition() - is_str = set(r.desc_with_module for r in result) - if is_str != should_str: - print('Solution @%s not right, received %s, wanted %s' \ - % (line_nr - 1, is_str, should_str)) - return 1 - return 0 - - -def run_goto_test(case): - """ - Tests look like this:: - - abc = 1 - #! ['abc=1'] - abc - - Additionally it is possible to add a number which describes to position of - the test (otherwise it's just end of line):: - - #! 2 ['abc=1'] - abc - - Returns 1 for fail and 0 for success. - """ - (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) - result = script.goto() - comp_str = str(sorted(str(r.description) for r in result)) - if comp_str != correct: - print('Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, comp_str, correct)) - return 1 - return 0 - - -def run_related_name_test(case): - """ - Tests look like this:: - - abc = 1 - #< abc@1,0 abc@3,0 - abc - - Returns 1 for fail and 0 for success. - """ - (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) - result = script.related_names() - correct = correct.strip() - compare = sorted((r.module_name, r.start_pos[0], r.start_pos[1]) - for r in result) - wanted = [] - if not correct: - positions = [] - else: - positions = literal_eval(correct) - for pos_tup in positions: - if type(pos_tup[0]) == str: - # this means that there is a module specified - wanted.append(pos_tup) - else: - wanted.append(('renaming', line_nr + pos_tup[0], pos_tup[1])) - - wanted = sorted(wanted) - if compare != wanted: - print('Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, compare, wanted)) - return 1 - return 0 - - class IntegrationTestCase(object): def __init__(self, test_type, correct, line_nr, column, start, line, @@ -290,86 +172,3 @@ def collect_dir_tests(base_dir, test_files, thirdparty=False): case.path = path case.source = source yield case - - -def run_test(cases): - """ - This is the completion test for some cases. The tests are not unit test - like, they are rather integration tests. - """ - testers = { - TEST_COMPLETIONS: run_completion_test, - TEST_DEFINITIONS: run_definition_test, - TEST_ASSIGNMENTS: run_goto_test, - TEST_USAGES: run_related_name_test, - } - - tests = 0 - fails = 0 - for case in cases: - tests += 1 - try: - fails += testers[case.test_type](case) - except Exception: - print(traceback.format_exc()) - print(case) - fails += 1 - return tests, fails - - -def test_dir(completion_test_dir, thirdparty=False): - for (path, cases) in itertools.groupby( - collect_dir_tests(completion_test_dir, test_files, thirdparty), - lambda case: case.path): - f_name = os.path.basename(path) - - if thirdparty: - lib = f_name.replace('_.py', '') - try: - # there is always an underline at the end. - # It looks like: completion/thirdparty/pylab_.py - __import__(lib) - except ImportError: - base.summary.append('Thirdparty-Library %s not found.' % - f_name) - continue - - num_tests, fails = run_test(cases) - base.test_sum += num_tests - - s = 'run %s tests with %s fails (%s)' % (num_tests, fails, f_name) - base.tests_fail += fails - print(s) - base.summary.append(s) - - -if __name__ == '__main__': - try: - i = sys.argv.index('--thirdparty') - thirdparty = True - sys.argv = sys.argv[:i] + sys.argv[i + 1:] - except ValueError: - thirdparty = False - - test_files = base.get_test_list() - - # completion tests: - completion_test_dir = os.path.join(base.test_dir, 'completion') - - # execute tests - test_dir(completion_test_dir) - if test_files or thirdparty: - completion_test_dir += '/thirdparty' - test_dir(completion_test_dir, thirdparty=True) - - base.print_summary() - #from guppy import hpy - #hpy() - #print hpy().heap() - - exit_code = 1 if base.tests_fail else 0 - if sys.hexversion < 0x02060000 and base.tests_fail <= 9: - # Python 2.5 has major incompabillities (e.g. no property.setter), - # therefore it is not possible to pass all tests. - exit_code = 0 - sys.exit(exit_code) diff --git a/test/test.sh b/test/test.sh deleted file mode 100755 index 4b41f743..00000000 --- a/test/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e - -python regression.py -python run.py -echo -python refactor.py -echo -nosetests --with-doctest --doctest-tests ../jedi/ From b5764c14461e1704daefaf8d9137ddcb3590dea9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:04:06 +0100 Subject: [PATCH 019/109] Remove old utility functions in test.base --- test/base.py | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/test/base.py b/test/base.py index 4277bb71..f01ca5cd 100644 --- a/test/base.py +++ b/test/base.py @@ -1,4 +1,3 @@ -import time import sys if sys.hexversion < 0x02070000: import unittest2 as unittest @@ -15,45 +14,8 @@ sys.path.insert(0, root_dir) import pytest import jedi -from jedi import debug from jedi._compatibility import is_py25 -test_sum = 0 -t_start = time.time() -# Sorry I didn't use argparse here. It's because argparse is not in the -# stdlib in 2.5. -args = sys.argv[1:] - -print_debug = False -try: - i = args.index('--debug') - args = args[:i] + args[i + 1:] -except ValueError: - pass -else: - print_debug = True - jedi.set_debug_function(debug.print_to_stdout) - -sys.argv = sys.argv[:1] + args - -summary = [] -tests_fail = 0 - - -def get_test_list(): -# get test list, that should be executed - test_files = {} - last = None - for arg in sys.argv[1:]: - if arg.isdigit(): - if last is None: - continue - test_files[last].append(int(arg)) - else: - test_files[arg] = [] - last = arg - return test_files - class TestBase(unittest.TestCase): def get_script(self, src, pos, path=None): @@ -79,13 +41,6 @@ class TestBase(unittest.TestCase): return script.function_definition() -def print_summary(): - print('\nSummary: (%s fails of %s tests) in %.3fs' % \ - (tests_fail, test_sum, time.time() - t_start)) - for s in summary: - print(s) - - def cwd_at(path): """ Decorator to run function at `path`. From 1fffbf13cae48954b8af4dc2f5c53afd0b18eafd Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:14:25 +0100 Subject: [PATCH 020/109] Fix test failures because imports.py uses base.py --- test/base.py | 3 +++ test/completion/imports.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/base.py b/test/base.py index f01ca5cd..a0e6666a 100644 --- a/test/base.py +++ b/test/base.py @@ -17,6 +17,9 @@ import jedi from jedi._compatibility import is_py25 +sample_int = 1 # This is used in completion/imports.py + + class TestBase(unittest.TestCase): def get_script(self, src, pos, path=None): if pos is None: diff --git a/test/completion/imports.py b/test/completion/imports.py index af73b99c..e651d97e 100644 --- a/test/completion/imports.py +++ b/test/completion/imports.py @@ -154,9 +154,9 @@ mod1.a from .. import base #? int() -base.tests_fail +base.sample_int -from ..base import tests_fail as f +from ..base import sample_int as f #? int() f From 446c7cf6943659d7e67db19d0084202cc0ad95e3 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:36:42 +0100 Subject: [PATCH 021/109] Document how to run test --- test/run.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/test/run.py b/test/run.py index 10cdf26d..ec2ac523 100755 --- a/test/run.py +++ b/test/run.py @@ -17,16 +17,36 @@ There are different kind of tests: How to run tests? +++++++++++++++++ -Basically ``run.py`` searches the ``completion`` directory for files with lines -starting with the symbol above. There is also support for third party -libraries. In a normal test run (``./run.py``) they are not being executed, you -have to provide a ``--thirdparty`` option. +Jedi uses pytest_ to run unit and integration tests. To run tests, +simply run ``py.test``. You can also use tox_ to run tests for +multiple Python versions. -Now it's much more important, that you know how test only one file (``./run.py -classes``, where ``classes`` is the name of the file to test) or even one test -(``./run.py classes 90``, which would just execute the test on line 90). +.. _pytest: http://pytest.org +.. _tox: http://testrun.org/tox -If you want to debug a test, just use the --debug option. +Integration test cases are located in ``test/completion`` directory +and each test cases are indicated by the comment ``#?`` (complete / +definitions), ``#!`` (assignments) and ``#<`` (usages). There is also +support for third party libraries. In a normal test run they are not +being executed, you have to provide a ``--thirdparty`` option. + +In addition to standard `-k` and `-m` options in py.test, you can use +`-T` (`--test-files`) option to specify integration test cases to run. +It takes the format of ``FILE_NAME[:LINE[,LINE[,...]]]`` where +``FILE_NAME`` is a file in ``test/completion`` and ``LINE`` is a line +number of the test comment. Here is some recipes: + +Run tests only in ``basic.py`` and ``imports.py``:: + + py.test test/test_integration.py -T basic.py -T imports.py + +Run test at line 4, 6, and 8 in ``basic.py``:: + + py.test test/test_integration.py -T basic.py:4,6,8 + +See ``py.test --help`` for more information. + +If you want to debug a test, just use the --pdb option. Auto-Completion +++++++++++++++ From c387bf06bcc72c4b4af7a2a24baa80a273c7d8d9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:03:12 +0100 Subject: [PATCH 022/109] No need to setup import path in test.base py.test does that for us. --- test/base.py | 1 - test/refactor.py | 1 - test/run.py | 5 ----- 3 files changed, 7 deletions(-) diff --git a/test/base.py b/test/base.py index a0e6666a..efdd9f18 100644 --- a/test/base.py +++ b/test/base.py @@ -9,7 +9,6 @@ import functools test_dir = dirname(abspath(__file__)) root_dir = dirname(test_dir) -sys.path.insert(0, root_dir) import pytest diff --git a/test/refactor.py b/test/refactor.py index 206e1662..05a8d154 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -10,7 +10,6 @@ import traceback import re import itertools -from . import base # required to setup import path from jedi._compatibility import reduce import jedi from jedi import refactoring diff --git a/test/run.py b/test/run.py index ec2ac523..e37abab2 100755 --- a/test/run.py +++ b/test/run.py @@ -97,17 +97,12 @@ Tests look like this:: abc """ import os -import sys import re -from . import base # required to setup import path import jedi from jedi._compatibility import unicode, StringIO, reduce, is_py25 -sys.path.pop(0) # pop again, because it might affect the completion - - TEST_COMPLETIONS = 0 TEST_DEFINITIONS = 1 TEST_ASSIGNMENTS = 2 From c87d3dad52d5beb65afae129a68dc8e9cf34bf3c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:04:15 +0100 Subject: [PATCH 023/109] Remove unused imports --- test/base.py | 7 ++++--- test/refactor.py | 3 --- test/test_integration.py | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/base.py b/test/base.py index efdd9f18..2deb4452 100644 --- a/test/base.py +++ b/test/base.py @@ -7,15 +7,16 @@ import os from os.path import abspath, dirname import functools -test_dir = dirname(abspath(__file__)) -root_dir = dirname(test_dir) - import pytest import jedi from jedi._compatibility import is_py25 +test_dir = dirname(abspath(__file__)) +root_dir = dirname(test_dir) + + sample_int = 1 # This is used in completion/imports.py diff --git a/test/refactor.py b/test/refactor.py index 05a8d154..fdbbdc20 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -4,11 +4,8 @@ Refactoring tests work a little bit similar to Black Box tests. But the idea is here to compare two versions of code. """ from __future__ import with_statement -import sys import os -import traceback import re -import itertools from jedi._compatibility import reduce import jedi diff --git a/test/test_integration.py b/test/test_integration.py index b3baf336..f348689d 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,8 +1,6 @@ import os import re -import pytest - from . import base from .run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES From de7092d56b5232c29b6b10e6d3df1098e2ee8f8c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:24:48 +0100 Subject: [PATCH 024/109] Fix: --thirdparty was not considered --- test/conftest.py | 10 ++++++---- test/run.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 4cb2f4e5..87a7fa9f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -25,7 +25,7 @@ def pytest_addoption(parser): "For example: -T generators.py:10,13,19. " "Note that you can use -m to specify the test case by id.")) parser.addoption( - "--thirdparty", + "--thirdparty", action='store_true', help="Include integration tests that requires third party modules.") @@ -53,9 +53,11 @@ def pytest_generate_tests(metafunc): if 'case' in metafunc.fixturenames: base_dir = metafunc.config.option.integration_case_dir thirdparty = metafunc.config.option.thirdparty - metafunc.parametrize( - 'case', - run.collect_dir_tests(base_dir, test_files, thirdparty)) + cases = list(run.collect_dir_tests(base_dir, test_files)) + if thirdparty: + cases.extend(run.collect_dir_tests( + os.path.join(base_dir, 'thirdparty'), test_files)) + metafunc.parametrize('case', cases) if 'refactor_case' in metafunc.fixturenames: base_dir = metafunc.config.option.refactor_case_dir metafunc.parametrize( diff --git a/test/run.py b/test/run.py index e37abab2..16d08f8a 100755 --- a/test/run.py +++ b/test/run.py @@ -171,7 +171,7 @@ def collect_file_tests(lines, lines_to_execute): correct = None -def collect_dir_tests(base_dir, test_files, thirdparty=False): +def collect_dir_tests(base_dir, test_files): for f_name in os.listdir(base_dir): files_to_execute = [a for a in test_files.items() if a[0] in f_name] lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, []) From e5b660b0a1d92cc9ce8e3ebf439f782fc6517108 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:33:01 +0100 Subject: [PATCH 025/109] Use PIP_INSECURE=t only in Python 2.5 test --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a5adc19..506376d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,9 @@ python: - 2.7 - 3.2 install: - # Needed only for Python 2.5: - - export PIP_INSECURE=t + - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then + export PIP_INSECURE=t; + fi - pip install --use-mirrors tox script: - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | From 4866b38bdafef4f0818594c06a2adfe0434852de Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 13:08:47 +0100 Subject: [PATCH 026/109] Make VIRTUAL_ENV actually importable --- jedi/modules.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jedi/modules.py b/jedi/modules.py index 844a1eb9..3ca6596b 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -279,9 +279,8 @@ def get_sys_path(): venv, 'lib', 'python%d.%d' % sys.version_info[:2], 'site-packages') sys_path.insert(0, p) - p = sys.path[1:] - check_virtual_env(p) - return p + check_virtual_env(sys.path) + return [p for p in sys.path if p != ""] @cache.memoize_default([]) From ab5266b8401e781b3b6cfc28f0f90ef6186a5d2a Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 13:49:20 +0100 Subject: [PATCH 027/109] Automatically skip thirdparty test if not importable --- test/conftest.py | 2 +- test/run.py | 16 +++++++++++++++- test/test_integration.py | 4 ++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 87a7fa9f..c5a00879 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -56,7 +56,7 @@ def pytest_generate_tests(metafunc): cases = list(run.collect_dir_tests(base_dir, test_files)) if thirdparty: cases.extend(run.collect_dir_tests( - os.path.join(base_dir, 'thirdparty'), test_files)) + os.path.join(base_dir, 'thirdparty'), test_files, True)) metafunc.parametrize('case', cases) if 'refactor_case' in metafunc.fixturenames: base_dir = metafunc.config.option.refactor_case_dir diff --git a/test/run.py b/test/run.py index 16d08f8a..2b24c72b 100755 --- a/test/run.py +++ b/test/run.py @@ -120,6 +120,7 @@ class IntegrationTestCase(object): self.start = start self.line = line self.path = path + self.skip = None def __repr__(self): name = os.path.basename(self.path) if self.path else None @@ -171,7 +172,7 @@ def collect_file_tests(lines, lines_to_execute): correct = None -def collect_dir_tests(base_dir, test_files): +def collect_dir_tests(base_dir, test_files, check_thirdparty=False): for f_name in os.listdir(base_dir): files_to_execute = [a for a in test_files.items() if a[0] in f_name] lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, []) @@ -180,10 +181,23 @@ def collect_dir_tests(base_dir, test_files): # only has these features partially. if is_py25 and f_name in ['generators.py', 'types.py']: continue + + skip = None + if check_thirdparty: + lib = f_name.replace('_.py', '') + try: + # there is always an underline at the end. + # It looks like: completion/thirdparty/pylab_.py + __import__(lib) + except ImportError: + skip = 'Thirdparty-Library %s not found.' % lib + path = os.path.join(base_dir, f_name) source = open(path).read() for case in collect_file_tests(StringIO(source), lines_to_execute): case.path = path case.source = source + if skip: + case.skip = skip yield case diff --git a/test/test_integration.py b/test/test_integration.py index f348689d..e522f721 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,6 +1,8 @@ import os import re +import pytest + from . import base from .run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES @@ -97,6 +99,8 @@ def run_related_name_test(case): def test_integration(case, monkeypatch, pytestconfig): + if case.skip is not None: + pytest.skip(case.skip) repo_root = base.root_dir monkeypatch.chdir(os.path.join(repo_root, 'jedi')) testers = { From c8c26f52b91c3d86f52f4e3ac245c2ae1648cab9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 19:35:20 +0100 Subject: [PATCH 028/109] Fix ImportError when building Sphinx --- docs/docs/testing.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/testing.rst b/docs/docs/testing.rst index e04a35b5..509fc76e 100644 --- a/docs/docs/testing.rst +++ b/docs/docs/testing.rst @@ -14,10 +14,10 @@ Blackbox Tests (run.py) .. automodule:: test.run -Regression Tests (regression.py) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Regression Tests (test_regression.py) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. automodule:: test.regression +.. automodule:: test.test_regression Refactoring Tests (refactor.py) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From f9ed3d42f7334cfd15e9a6eb098093fe16564224 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 20:01:43 +0100 Subject: [PATCH 029/109] Document BaseDefinition.type --- jedi/api_classes.py | 48 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index a84c7fe8..820792d0 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -69,7 +69,53 @@ class BaseDefinition(object): @property def type(self): - """The type of the definition.""" + """ + The type of the definition. + + Here is an example of the value of this attribute. Let's consider + the following source. As what is in ``variable`` is unambiguous + to Jedi, :meth:`api.Script.definition` should return a list of + definition for ``sys``, ``f``, ``C`` and ``x``. + + >>> from jedi import Script + >>> source = ''' + ... import sys + ... + ... class C: + ... pass + ... + ... class D: + ... pass + ... + ... x = D() + ... + ... def f(): + ... pass + ... + ... variable = sys or f or C or x''' + >>> script = Script(source, len(source.splitlines()), 3, 'example.py') + >>> defs = script.definition() + + Before showing what is in ``defs``, let's sort it by :attr:`line` + so that it is easy to relate the result to the source code. + + >>> defs = sorted(defs, key=lambda d: d.line) + >>> defs # doctest: +NORMALIZE_WHITESPACE + [, , + , ] + + Finally, here is what you can get from :attr:`type`: + + >>> defs[0].type + 'Module' + >>> defs[1].type + 'Class' + >>> defs[2].type + 'Instance' + >>> defs[3].type + 'Function' + + """ # generate the type stripped = self.definition if isinstance(self.definition, er.InstanceElement): From 41b33fd460f01f39e7278701dd49878fbbe302a2 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 20:11:14 +0100 Subject: [PATCH 030/109] Document BaseDefinition.doc --- jedi/api_classes.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 820792d0..95cbdd41 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -171,7 +171,31 @@ class BaseDefinition(object): @property def doc(self): - """Return a document string for this completion object.""" + r""" + Return a document string for this completion object. + + Example: + + >>> from jedi import Script + >>> source = '''\ + ... def f(a, b=1): + ... "Document for function f." + ... ''' + >>> script = Script(source, 1, len('def f'), 'example.py') + >>> d = script.definition()[0] + >>> print(d.doc) + f(a, b = 1) + + Document for function f. + + Notice that useful extra information is added to the actual + docstring. For function, it is call signature. If you need + actual docstring, use :attr:`raw_doc` instead. + + >>> print(d.raw_doc) + Document for function f. + + """ try: return self.definition.doc except AttributeError: @@ -179,7 +203,11 @@ class BaseDefinition(object): @property def raw_doc(self): - """The raw docstring ``__doc__`` for any object.""" + """ + The raw docstring ``__doc__`` for any object. + + See :attr:`doc` for example. + """ try: return unicode(self.definition.docstr) except AttributeError: From 9e3e6a2eeae4d8d2fd038f4ca480b832aa1cc1a5 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 20:29:02 +0100 Subject: [PATCH 031/109] Document BaseDefinition.description --- jedi/api_classes.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 95cbdd41..7feca504 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -215,7 +215,31 @@ class BaseDefinition(object): @property def description(self): - """A textual description of the object.""" + """ + A textual description of the object. + + Example: + + >>> from jedi import Script + >>> source = ''' + ... def f(): + ... pass + ... + ... class C: + ... pass + ... + ... variable = f or C''' + >>> script = Script(source, len(source.splitlines()), 3, 'example.py') + >>> defs = script.definition() + >>> defs = sorted(defs, key=lambda d: d.line) + >>> defs + [, ] + >>> defs[0].description + 'def f' + >>> defs[1].description + 'class C' + + """ return unicode(self.definition) @property From a449428391296d11e1b001704fc4d650ba58e464 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 20:30:36 +0100 Subject: [PATCH 032/109] Document BaseDefinition.full_name --- jedi/api_classes.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 7feca504..1647fdd9 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -244,7 +244,27 @@ class BaseDefinition(object): @property def full_name(self): - """The path to a certain class/function, see #61.""" + """ + Dot-separated path of this object. + + It is in the form of ``[.[...]][.]``. + It is useful when you want to look up Python manual of the + object at hand. + + Example: + + >>> from jedi import Script + >>> source = ''' + ... import os + ... os.path.join''' + >>> script = Script(source, 3, len('os.path.join'), 'example.py') + >>> print(script.definition()[0].full_name) + os.path.join + + Notice that it correctly returns ``'os.path.join'`` instead of + (for example) ``'posixpath.join'``. + + """ path = [unicode(p) for p in self.path] # TODO add further checks, the mapping should only occur on stdlib. if not path: From 2f7797f86707ea129ac85b32a0fda3b28f83084e Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 21:44:42 +0100 Subject: [PATCH 033/109] Ignore known failure in BaseDefinition.description doctest See: #162 --- jedi/api_classes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 1647fdd9..cf90c5b4 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -230,13 +230,13 @@ class BaseDefinition(object): ... ... variable = f or C''' >>> script = Script(source, len(source.splitlines()), 3, 'example.py') - >>> defs = script.definition() - >>> defs = sorted(defs, key=lambda d: d.line) - >>> defs + >>> defs = script.definition() # doctest: +SKIP + >>> defs = sorted(defs, key=lambda d: d.line) # doctest: +SKIP + >>> defs # doctest: +SKIP [, ] - >>> defs[0].description + >>> defs[0].description # doctest: +SKIP 'def f' - >>> defs[1].description + >>> defs[1].description # doctest: +SKIP 'class C' """ From 125a3b1a6b1eba644fd47ad239414abdb764f6c8 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 21:57:00 +0100 Subject: [PATCH 034/109] Document BaseDefinition.module_name --- jedi/api_classes.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index cf90c5b4..92ba78a0 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -138,7 +138,17 @@ class BaseDefinition(object): @property def module_name(self): - """The module name.""" + """ + The module name. + + >>> from jedi import Script + >>> source = 'import datetime' + >>> script = Script(source, 1, len(source), 'example.py') + >>> d = script.definition()[0] + >>> print(d.module_name) # doctest: +ELLIPSIS + datetime + + """ path = self.module_path sep = os.path.sep p = re.sub(r'^.*?([\w\d]+)(%s__init__)?.py$' % sep, r'\1', path) From 8d54ebea7b5f63a77b6adeffdbe40a4599f0cb24 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 22:05:59 +0100 Subject: [PATCH 035/109] Fix BaseDefinition.module_name --- jedi/api_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 92ba78a0..9d659467 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -151,7 +151,7 @@ class BaseDefinition(object): """ path = self.module_path sep = os.path.sep - p = re.sub(r'^.*?([\w\d]+)(%s__init__)?.py$' % sep, r'\1', path) + p = re.sub(r'^.*?([\w\d]+)(%s__init__)?.(py|so)$' % sep, r'\1', path) return p def in_builtin_module(self): From dccda224ab4e267a603e505681b0e61beaab663f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 22:17:25 +0100 Subject: [PATCH 036/109] Hide api_classes.defined_names --- jedi/api.py | 2 +- jedi/api_classes.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 00a39691..9de6d8ac 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -507,7 +507,7 @@ def defined_names(source, source_path=None, source_encoding='utf-8'): modules.source_to_unicode(source, source_encoding), module_path=source_path, ) - return api_classes.defined_names(parser.scope) + return api_classes._defined_names(parser.scope) def set_debug_function(func_cb=debug.print_to_stdout, warnings=True, diff --git a/jedi/api_classes.py b/jedi/api_classes.py index a84c7fe8..67070824 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -365,10 +365,10 @@ class Definition(BaseDefinition): d = d.var if isinstance(d, pr.Name): d = d.parent - return defined_names(d) + return _defined_names(d) -def defined_names(scope): +def _defined_names(scope): """ List sub-definitions (e.g., methods in class). From e42ff9e762dcf3e3fec21bd061bb787a1326a871 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 23:21:15 +0100 Subject: [PATCH 037/109] Add common.rethrow_uncaught --- jedi/_compatibility.py | 10 ++++++++++ jedi/common.py | 15 ++++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 78abf874..5f03f736 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -81,6 +81,16 @@ else: eval(compile("""def exec_function(source, global_map): exec source in global_map """, 'blub', 'exec')) +# re-raise function +if is_py3k: + def reraise(exception, traceback): + raise exception.with_traceback(traceback) +else: + eval(compile(""" +def reraise(exception, traceback): + raise exception, None, traceback +""", 'blub', 'exec')) + # StringIO (Python 2.5 has no io module), so use io only for py3k try: from StringIO import StringIO diff --git a/jedi/common.py b/jedi/common.py index 35a10f62..83337eef 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -1,8 +1,10 @@ """ A universal module with functions / classes without dependencies. """ +import sys import contextlib +import functools import tokenize -from _compatibility import next +from _compatibility import next, reraise import debug import settings @@ -33,6 +35,17 @@ class MultiLevelAttributeError(Exception): return 'Original:\n\n' + ''.join(tb) +def rethrow_uncaught(func): + @functools.wraps(func) + def wrapper(*args, **kwds): + try: + return func(*args, **kwds) + except AttributeError: + exc_info = sys.exc_info() + reraise(MultiLevelAttributeError, exc_info[2]) + return wrapper + + class PushBackIterator(object): def __init__(self, iterator): self.pushes = [] From eca0f01cfb79a718834574413d0d117783b110a9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 23:21:49 +0100 Subject: [PATCH 038/109] Use @common.rethrow_uncaught and stop manual re-raise --- jedi/evaluate.py | 8 ++--- jedi/evaluate_representation.py | 55 +++++++++++++++------------------ 2 files changed, 27 insertions(+), 36 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 8d68b050..3ad076bc 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -581,12 +581,7 @@ def follow_statement(stmt, seek_name=None): commands = stmt.get_commands() debug.dbg('calls: %s' % commands) - try: - result = follow_call_list(commands) - except AttributeError: - # This is so evil! But necessary to propagate errors. The attribute - # errors here must not be catched, because they shouldn't exist. - raise common.MultiLevelAttributeError(sys.exc_info()) + result = follow_call_list(commands) # Assignment checking is only important if the statement defines multiple # variables. @@ -598,6 +593,7 @@ def follow_statement(stmt, seek_name=None): return set(result) +@common.rethrow_uncaught def follow_call_list(call_list, follow_array=False): """ `call_list` can be either `pr.Array` or `list of list`. diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 82d2a034..0b5995e7 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -9,7 +9,6 @@ instantiated. This class represents these cases. So, why is there also a ``Class`` class here? Well, there are decorators and they change classes in Python 3. """ -import sys import copy import itertools @@ -668,6 +667,7 @@ class Execution(Executable): """ return self.get_params() + pr.Scope.get_set_vars(self) + @common.rethrow_uncaught def copy_properties(self, prop): """ Literally copies a property of a Function. Copying is very expensive, @@ -675,22 +675,19 @@ class Execution(Executable): objects can be used for the executions, as if they were in the execution. """ - try: - # Copy all these lists into this local function. - attr = getattr(self.base, prop) - objects = [] - for element in attr: - if element is None: - copied = element - else: - copied = helpers.fast_parent_copy(element) - copied.parent = self._scope_copy(copied.parent) - if isinstance(copied, pr.Function): - copied = Function(copied) - objects.append(copied) - return objects - except AttributeError: - raise common.MultiLevelAttributeError(sys.exc_info()) + # Copy all these lists into this local function. + attr = getattr(self.base, prop) + objects = [] + for element in attr: + if element is None: + copied = element + else: + copied = helpers.fast_parent_copy(element) + copied.parent = self._scope_copy(copied.parent) + if isinstance(copied, pr.Function): + copied = Function(copied) + objects.append(copied) + return objects def __getattr__(self, name): if name not in ['start_pos', 'end_pos', 'imports', '_sub_module']: @@ -698,21 +695,19 @@ class Execution(Executable): return getattr(self.base, name) @cache.memoize_default() + @common.rethrow_uncaught def _scope_copy(self, scope): - try: - """ Copies a scope (e.g. if) in an execution """ - # TODO method uses different scopes than the subscopes property. + """ Copies a scope (e.g. if) in an execution """ + # TODO method uses different scopes than the subscopes property. - # just check the start_pos, sometimes it's difficult with closures - # to compare the scopes directly. - if scope.start_pos == self.start_pos: - return self - else: - copied = helpers.fast_parent_copy(scope) - copied.parent = self._scope_copy(copied.parent) - return copied - except AttributeError: - raise common.MultiLevelAttributeError(sys.exc_info()) + # just check the start_pos, sometimes it's difficult with closures + # to compare the scopes directly. + if scope.start_pos == self.start_pos: + return self + else: + copied = helpers.fast_parent_copy(scope) + copied.parent = self._scope_copy(copied.parent) + return copied @property @cache.memoize_default() From 3b78b52204de48e09e726f5c1d15480bf846ee79 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 23:24:26 +0100 Subject: [PATCH 039/109] Rename MultiLevelAttributeErro to UncaughtAttributeError --- jedi/common.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jedi/common.py b/jedi/common.py index 83337eef..37d3bb21 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -16,7 +16,7 @@ class MultiLevelStopIteration(Exception): pass -class MultiLevelAttributeError(Exception): +class UncaughtAttributeError(Exception): """ Important, because `__getattr__` and `hasattr` catch AttributeErrors implicitly. This is really evil (mainly because of `__getattr__`). @@ -42,7 +42,7 @@ def rethrow_uncaught(func): return func(*args, **kwds) except AttributeError: exc_info = sys.exc_info() - reraise(MultiLevelAttributeError, exc_info[2]) + reraise(UncaughtAttributeError, exc_info[2]) return wrapper From 2b89dda5a67f1bf65f8b5890b2957f2226705c22 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 23:28:53 +0100 Subject: [PATCH 040/109] Use reraise when re-raising MultiLevelStopIteration --- jedi/evaluate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 3ad076bc..9c4e30c6 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -68,7 +68,7 @@ backtracking algorithm. .. todo:: nonlocal statement, needed or can be ignored? (py3k) """ -from _compatibility import next, hasattr, is_py3k, unicode, utf8 +from _compatibility import next, hasattr, is_py3k, unicode, reraise import sys import itertools @@ -175,7 +175,7 @@ def get_names_of_scope(scope, position=None, star_search=True, yield scope, get_defined_names_for_position(scope, position, in_func_scope) except StopIteration: - raise common.MultiLevelStopIteration('StopIteration raised') + reraise(common.MultiLevelStopIteration, sys.exc_info()[2]) if scope.isinstance(pr.ForFlow) and scope.is_list_comp: # is a list comprehension yield scope, scope.get_set_vars(is_internal_call=True) From 93bd00bba438142f11120ada9161dff28ed4320f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 23:49:51 +0100 Subject: [PATCH 041/109] Document rethrow_uncaught/reraise --- jedi/_compatibility.py | 9 +++++++++ jedi/common.py | 24 ++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 5f03f736..9a704304 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -91,6 +91,15 @@ def reraise(exception, traceback): raise exception, None, traceback """, 'blub', 'exec')) +reraise.__doc__ = """ +Re-raise `exception` with a `traceback` object. + +Usage:: + + reraise(Exception, sys.exc_info()[2]) + +""" + # StringIO (Python 2.5 has no io module), so use io only for py3k try: from StringIO import StringIO diff --git a/jedi/common.py b/jedi/common.py index 37d3bb21..62b88220 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -21,8 +21,9 @@ class UncaughtAttributeError(Exception): Important, because `__getattr__` and `hasattr` catch AttributeErrors implicitly. This is really evil (mainly because of `__getattr__`). `hasattr` in Python 2 is even more evil, because it catches ALL exceptions. - Therefore this class has to be a `BaseException` and not an `Exception`. - But because I rewrote hasattr, we can now switch back to `Exception`. + Therefore this class originally had to be derived from `BaseException` + instead of `Exception`. But because I removed relevant `hasattr` from + the code base, we can now switch back to `Exception`. :param base: return values of sys.exc_info(). """ @@ -36,6 +37,25 @@ class UncaughtAttributeError(Exception): def rethrow_uncaught(func): + """ + Re-throw uncaught `AttributeError`. + + Usage: Put ``@rethrow_uncaught`` in front of the function + which does **not** suppose to raise `AttributeError`. + + AttributeError is easily get caught by `hasattr` and another + ``except AttributeError`` clause. This becomes problem when you use + a lot of "dynamic" attributes (e.g., using ``@property``) because you + can't distinguish if the property does not exist for real or some code + inside of the "dynamic" attribute through that error. In a well + written code, such error should not exist but getting there is very + difficult. This decorator is to help us getting there by changing + `AttributeError` to `UncaughtAttributeError` to avoid unexpected catch. + This helps us noticing bugs earlier and facilitates debugging. + + .. note:: Treating StopIteration here is easy. + Add that feature when needed. + """ @functools.wraps(func) def wrapper(*args, **kwds): try: From 7a9e374a656d5fa1a12a8e4ed8195d7361aa0812 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 23:54:19 +0100 Subject: [PATCH 042/109] Use original exception value when reraise --- jedi/common.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/jedi/common.py b/jedi/common.py index 62b88220..d50db623 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -27,13 +27,6 @@ class UncaughtAttributeError(Exception): :param base: return values of sys.exc_info(). """ - def __init__(self, base=None): - self.base = base - - def __str__(self): - import traceback - tb = traceback.format_exception(*self.base) - return 'Original:\n\n' + ''.join(tb) def rethrow_uncaught(func): @@ -62,7 +55,7 @@ def rethrow_uncaught(func): return func(*args, **kwds) except AttributeError: exc_info = sys.exc_info() - reraise(UncaughtAttributeError, exc_info[2]) + reraise(UncaughtAttributeError(exc_info[1]), exc_info[2]) return wrapper From 191e8f690b603f8fc7a1539ca6e1ec53a311c162 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 10:33:28 +0100 Subject: [PATCH 043/109] Simplify .travis.yml --- .travis.yml | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 506376d3..1c14b5f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,10 @@ language: python -python: - - 2.5 - - 2.6 - - 2.7 - - 3.2 +env: + - TOXENV=py25 PIP_INSECURE=t + - TOXENV=py26 + - TOXENV=py27 + - TOXENV=py32 install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - export PIP_INSECURE=t; - fi - pip install --use-mirrors tox script: - - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | - sed --regexp-extended 's/([0-9])\.([0-9])/py\1\2/g') - - echo "TOXENV=$TOXENV" - tox From 3d647d3f3dd765a904526bac2a4598617255c2f4 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 10:35:20 +0100 Subject: [PATCH 044/109] Use --quiet when installing tox --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1c14b5f4..e1529542 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,6 @@ env: - TOXENV=py27 - TOXENV=py32 install: - - pip install --use-mirrors tox + - pip install --quiet --use-mirrors tox script: - tox From fbccdd3f70ceea8eb465c2248a7297411c999097 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 14:15:00 +0100 Subject: [PATCH 045/109] Run doctests in the main py.test run --- pytest.ini | 5 ++++- tox.ini | 3 --- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pytest.ini b/pytest.ini index 4afbf164..5f33ed4a 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,8 @@ [pytest] -addopts = --assert=plain +addopts = --assert=plain --doctest-modules + +# Ignore broken files in blackbox test directories +norecursedirs = .* completion refactor # Activate `clean_jedi_cache` fixture for all tests. This should be # fine as long as we are using `clean_jedi_cache` as a session scoped diff --git a/tox.ini b/tox.ini index 25014940..6b2bad01 100644 --- a/tox.ini +++ b/tox.ini @@ -5,9 +5,6 @@ deps = pytest commands = py.test [] - # Doctests can't be run with the main tests because then py.test - # tries to import broken python files under test/*/. - py.test --doctest-modules {toxinidir}/jedi [testenv:py25] deps = simplejson From 3881f8d6edb56b725cb599f90251ad89c01af6cb Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 14:31:32 +0100 Subject: [PATCH 046/109] Do not collect setup.py as a test --- conftest.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 conftest.py diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..044963bb --- /dev/null +++ b/conftest.py @@ -0,0 +1 @@ +collect_ignore = ["setup.py"] From 95aed3c54e97222bbf4e4ffc435b6998fee84322 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 19:20:20 +0100 Subject: [PATCH 047/109] Exclude docs directory from test search --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 5f33ed4a..2be72740 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,7 +2,7 @@ addopts = --assert=plain --doctest-modules # Ignore broken files in blackbox test directories -norecursedirs = .* completion refactor +norecursedirs = .* docs completion refactor # Activate `clean_jedi_cache` fixture for all tests. This should be # fine as long as we are using `clean_jedi_cache` as a session scoped From ee03f3ae0d0501568cec87d8d4d7114441c19776 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 20:34:02 +0100 Subject: [PATCH 048/109] Use pytest_(un)configure to setup cache_directory --- conftest.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/conftest.py b/conftest.py index 044963bb..b9e86425 100644 --- a/conftest.py +++ b/conftest.py @@ -1 +1,34 @@ +import tempfile +import shutil + +import jedi + + collect_ignore = ["setup.py"] + + +# The following hooks (pytest_configure, pytest_unconfigure) are used +# to modify `jedi.settings.cache_directory` because `clean_jedi_cache` +# has no effect during doctests. Without these hooks, doctests uses +# user's cache (e.g., ~/.cache/jedi/). We should remove this +# workaround once the problem is fixed in py.test. +# +# See: +# - https://github.com/davidhalter/jedi/pull/168 +# - https://bitbucket.org/hpk42/pytest/issue/275/ + +jedi_cache_directory_orig = None +jedi_cache_directory_temp = None + + +def pytest_configure(config): + global jedi_cache_directory_orig, jedi_cache_directory_temp + jedi_cache_directory_orig = jedi.settings.cache_directory + jedi_cache_directory_temp = tempfile.mkdtemp(prefix='jedi-test-') + jedi.settings.cache_directory = jedi_cache_directory_temp + + +def pytest_unconfigure(config): + global jedi_cache_directory_orig, jedi_cache_directory_temp + jedi.settings.cache_directory = jedi_cache_directory_orig + shutil.rmtree(jedi_cache_directory_temp) From 860cf7b9743744c9d21796b227cf21d684fb5519 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 21:33:36 +0100 Subject: [PATCH 049/109] Add test_modulepickling_change_cache_dir --- test/test_cache.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 test/test_cache.py diff --git a/test/test_cache.py b/test/test_cache.py new file mode 100644 index 00000000..9f845fc2 --- /dev/null +++ b/test/test_cache.py @@ -0,0 +1,30 @@ +from jedi import settings +from jedi.cache import ParserCacheItem, _ModulePickling + + +ModulePickling = _ModulePickling() + + +def test_modulepickling_change_cache_dir(monkeypatch, tmpdir): + """ + ModulePickling should not save old cache when cache_directory is changed. + + See: `#168 `_ + """ + dir_1 = str(tmpdir.mkdir('first')) + dir_2 = str(tmpdir.mkdir('second')) + + item_1 = ParserCacheItem('fake parser 1') + item_2 = ParserCacheItem('fake parser 2') + path_1 = 'fake path 1' + path_2 = 'fake path 2' + + monkeypatch.setattr(settings, 'cache_directory', dir_1) + ModulePickling.save_module(path_1, item_1) + cached = ModulePickling.load_module(path_1, item_1.change_time - 1) + assert cached == item_1.parser + + monkeypatch.setattr(settings, 'cache_directory', dir_2) + ModulePickling.save_module(path_2, item_2) + cached = ModulePickling.load_module(path_1, item_1.change_time - 1) + assert cached is None From edc7148320454ea3ef1e7d5a2c42c132abf39234 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 22:15:34 +0100 Subject: [PATCH 050/109] Fix test_modulepickling_change_cache_dir failure --- jedi/cache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jedi/cache.py b/jedi/cache.py index e60583f4..84d9ede7 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -238,6 +238,7 @@ class _ModulePickling(object): return parser_cache_item.parser def save_module(self, path, parser_cache_item): + self.__index = None try: files = self._index[self.py_version] except KeyError: From 841d46684f4083af39c61e3ba4384535f98a3974 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Thu, 14 Mar 2013 22:21:58 +0100 Subject: [PATCH 051/109] Skip test in Python 3. It is a known bug. See: #161 --- test/test_cache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_cache.py b/test/test_cache.py index 9f845fc2..8352fd11 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -1,3 +1,5 @@ +import pytest + from jedi import settings from jedi.cache import ParserCacheItem, _ModulePickling @@ -5,6 +7,7 @@ from jedi.cache import ParserCacheItem, _ModulePickling ModulePickling = _ModulePickling() +@pytest.mark.skipif("sys.version_info >= (3,0)") def test_modulepickling_change_cache_dir(monkeypatch, tmpdir): """ ModulePickling should not save old cache when cache_directory is changed. From 7cf70a3f0a79ced8fb2e9e847dc52abe0b8af224 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 00:21:46 +0100 Subject: [PATCH 052/109] Do not clear cache in __del__ Prior to this change, running `py.test --assert=rewrite test/test_integration.py` fails with errors due to: ```py cls = @classmethod def cleanup(cls): > cls.parent_execution_funcs.pop() E IndexError: pop from empty list recursion.py:127: IndexError ``` Similar errors occurred in travis occasionally: - https://travis-ci.org/davidhalter/jedi/jobs/5449831 - https://travis-ci.org/davidhalter/jedi/jobs/5512538 I think this is because GC calls __del__ at random point during actual execution of ExecutionRecursionDecorator.__call__. As --assert=rewrite works by AST dynamically, I guess that it is harder for Python's GC to clean up code and therefore GC happens sometime later. Although this is just a random guess, as `tox -- --assert=rewrite` works with this patch now, I think this is a good change. --- jedi/api.py | 8 +++++--- jedi/api_classes.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 9de6d8ac..86b9a369 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -76,6 +76,7 @@ class Script(object): """ lazy parser.""" return self._module.parser + @api_classes._clear_caches_after_call def complete(self): """ Return :class:`api_classes.Completion` objects. Those objects contain @@ -209,6 +210,7 @@ class Script(object): warnings.warn("Use line instead.", DeprecationWarning) return self.definition() + @api_classes._clear_caches_after_call def definition(self): """ Return the definitions of a the path under the cursor. This is not a @@ -272,6 +274,7 @@ class Script(object): if not isinstance(s, imports.ImportPath._GlobalNamespace)]) return sorted(d, key=lambda x: (x.module_path, x.start_pos)) + @api_classes._clear_caches_after_call def goto(self): """ Return the first definition found by goto. Imports and statements @@ -334,6 +337,7 @@ class Script(object): definitions = [user_stmt] return definitions, search_name + @api_classes._clear_caches_after_call def related_names(self, additional_module_paths=()): """ Return :class:`api_classes.RelatedName` objects, which contain all @@ -378,6 +382,7 @@ class Script(object): warnings.warn("Use line instead.", DeprecationWarning) return self.function_definition() + @api_classes._clear_caches_after_call def function_definition(self): """ Return the function object of the call you're currently in. @@ -487,9 +492,6 @@ class Script(object): match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path, flags=re.S) return match.groups() - def __del__(self): - api_classes._clear_caches() - def defined_names(source, source_path=None, source_encoding='utf-8'): """ diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 051baa93..2be10641 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -7,6 +7,7 @@ interesting information about completion and goto operations. import re import os import warnings +import functools from _compatibility import unicode, next import cache @@ -34,6 +35,18 @@ def _clear_caches(): imports.imports_processed = 0 +def _clear_caches_after_call(func): + """ + Clear caches just before returning a value. + """ + @functools.wraps(func) + def wrapper(*args, **kwds): + result = func(*args, **kwds) + _clear_caches() + return result + return wrapper + + class BaseDefinition(object): _mapping = {'posixpath': 'os.path', 'riscospath': 'os.path', From 6659a532ac7047ec2f8dfaa93e0e7bda5754cdbb Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 00:25:10 +0100 Subject: [PATCH 053/109] Remove --assert=plain --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 2be72740..b393f082 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = --assert=plain --doctest-modules +addopts = --doctest-modules # Ignore broken files in blackbox test directories norecursedirs = .* docs completion refactor From 9ad9c806bbb5c605cb4c5d62543c52c7d4a2b94c Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 15 Mar 2013 15:02:33 +0430 Subject: [PATCH 054/109] rename non-cycling imports to 'from jedi import ..' --- jedi/api.py | 23 +++++++++++------------ jedi/api_classes.py | 12 ++++++------ jedi/builtin.py | 10 +++++----- jedi/cache.py | 6 +++--- jedi/common.py | 6 +++--- jedi/docstrings.py | 4 ++-- jedi/dynamic.py | 16 ++++++++-------- jedi/evaluate.py | 12 ++++++------ jedi/evaluate_representation.py | 16 ++++++++-------- jedi/fast_parser.py | 10 +++++----- jedi/helpers.py | 2 +- jedi/imports.py | 10 +++++----- jedi/keywords.py | 6 +++--- jedi/modules.py | 12 ++++++------ jedi/parsing.py | 6 +++--- jedi/parsing_representation.py | 6 +++--- jedi/recursion.py | 8 ++++---- jedi/refactoring.py | 8 ++++---- 18 files changed, 86 insertions(+), 87 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 86b9a369..429ec603 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -13,23 +13,22 @@ import re import os import warnings -import parsing -import parsing_representation as pr +from jedi import parsing +from jedi import parsing_representation as pr +from jedi import debug +from jedi import settings +from jedi import helpers +from jedi import common +from jedi import cache +from jedi import api_classes +from jedi import modules +from jedi import keywords +from jedi._compatibility import next, unicode import evaluate_representation as er import dynamic import imports import evaluate -import modules -import debug -import settings -import keywords -import helpers -import common import builtin -import api_classes -import cache - -from _compatibility import next, unicode class NotFoundError(Exception): diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 2be10641..25a4cf48 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -9,16 +9,16 @@ import os import warnings import functools -from _compatibility import unicode, next -import cache +from jedi._compatibility import unicode, next +from jedi import recursion +from jedi import settings +from jedi import parsing_representation as pr +from jedi import keywords +from jedi import cache import dynamic -import recursion -import settings import evaluate import imports -import parsing_representation as pr import evaluate_representation as er -import keywords def _clear_caches(): diff --git a/jedi/builtin.py b/jedi/builtin.py index b75d5b6b..73375e62 100644 --- a/jedi/builtin.py +++ b/jedi/builtin.py @@ -22,7 +22,7 @@ possible to access functions like ``list`` and ``int`` directly, the same way """ from __future__ import with_statement -from _compatibility import exec_function, is_py3k +from jedi._compatibility import exec_function, is_py3k import re import sys @@ -32,11 +32,11 @@ if is_py3k: import types import inspect -import common -import debug -import parsing +from jedi import common +from jedi import debug +from jedi import parsing +from jedi import modules import evaluate -import modules class BuiltinModule(modules.CachedModule): diff --git a/jedi/cache.py b/jedi/cache.py index 84d9ede7..2efc764d 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -23,9 +23,9 @@ import os import sys import pickle -from _compatibility import json -import settings -import debug +from jedi._compatibility import json +from jedi import settings +from jedi import debug # memoize caches will be deleted after every action memoize_caches = [] diff --git a/jedi/common.py b/jedi/common.py index d50db623..08eadd6b 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -4,9 +4,9 @@ import contextlib import functools import tokenize -from _compatibility import next, reraise -import debug -import settings +from jedi._compatibility import next, reraise +from jedi import debug +from jedi import settings class MultiLevelStopIteration(Exception): diff --git a/jedi/docstrings.py b/jedi/docstrings.py index e7d413b8..b3372e47 100644 --- a/jedi/docstrings.py +++ b/jedi/docstrings.py @@ -16,10 +16,10 @@ annotations. import re -import cache +from jedi import cache +from jedi import parsing import evaluate import evaluate_representation as er -import parsing DOCSTRING_PARAM_PATTERNS = [ r'\s*:type\s+%s:\s*([^\n]+)', # Sphinx diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 2d270a43..687c46da 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -55,16 +55,16 @@ from __future__ import with_statement import os -import cache -import parsing_representation as pr -import evaluate_representation as er -import modules +from jedi import cache +from jedi import parsing_representation as pr +from jedi import modules +from jedi import settings +from jedi import debug +from jedi import api_classes +from jedi import fast_parser import evaluate -import settings -import debug import imports -import api_classes -import fast_parser +import evaluate_representation as er # This is something like the sys.path, but only for searching params. It means # that this is the order in which Jedi searches params. diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 9c4e30c6..016e7075 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -73,16 +73,16 @@ from _compatibility import next, hasattr, is_py3k, unicode, reraise import sys import itertools -import common -import cache -import parsing_representation as pr +from jedi import common +from jedi import cache +from jedi import parsing_representation as pr +from jedi import debug +from jedi import recursion +import docstrings import evaluate_representation as er -import debug import builtin import imports -import recursion import dynamic -import docstrings def get_defined_names_for_position(scope, position=None, start_scope=None): diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 0b5995e7..31fba868 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -13,17 +13,17 @@ import copy import itertools from _compatibility import property, use_metaclass, next, hasattr -import parsing_representation as pr -import imports +from jedi import parsing_representation as pr +from jedi import cache +from jedi import helpers +from jedi import debug +from jedi import common +from jedi import recursion import docstrings -import cache +import imports +import evaluate import builtin import dynamic -import helpers -import recursion -import debug -import evaluate -import common class DecoratorNotFound(LookupError): diff --git a/jedi/fast_parser.py b/jedi/fast_parser.py index 8e14b953..4883e9c8 100644 --- a/jedi/fast_parser.py +++ b/jedi/fast_parser.py @@ -6,11 +6,11 @@ finished (and still not working as I want), I won't document it any further. import re import operator -from _compatibility import use_metaclass, reduce, property -import settings -import parsing -import parsing_representation as pr -import cache +from jedi._compatibility import use_metaclass, reduce, property +from jedi import settings +from jedi import parsing +from jedi import parsing_representation as pr +from jedi import cache class Module(pr.Simple, pr.Module): diff --git a/jedi/helpers.py b/jedi/helpers.py index bcc44125..c4532961 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -1,6 +1,6 @@ import copy -import parsing_representation as pr +from jedi import parsing_representation as pr def fast_parent_copy(obj): diff --git a/jedi/imports.py b/jedi/imports.py index 5a6353cb..90c119cd 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -20,13 +20,13 @@ import pkgutil import imp import sys +from jedi import modules +from jedi import debug +from jedi import parsing_representation as pr +from jedi import itertools +from jedi import cache import builtin -import modules -import debug -import parsing_representation as pr import evaluate -import itertools -import cache # for debugging purposes only imports_processed = 0 diff --git a/jedi/keywords.py b/jedi/keywords.py index 950270b7..07461bb7 100644 --- a/jedi/keywords.py +++ b/jedi/keywords.py @@ -1,9 +1,9 @@ +import pydoc import keyword -from _compatibility import is_py3k -import builtin +from jedi._compatibility import is_py3k +from jedi import builtin -import pydoc try: from pydoc_data import topics as pydoc_topics except ImportError: diff --git a/jedi/modules.py b/jedi/modules.py index 3ca6596b..b6c8f1e8 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -22,12 +22,12 @@ import tokenize import sys import os -import cache -import parsing -import parsing_representation as pr -import fast_parser -import debug -import settings +from jedi import cache +from jedi import parsing +from jedi import parsing_representation as pr +from jedi import fast_parser +from jedi import debug +from jedi import settings class CachedModule(object): diff --git a/jedi/parsing.py b/jedi/parsing.py index df52bf94..17ca7d49 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -20,9 +20,9 @@ from _compatibility import next, StringIO import tokenize import keyword -import debug -import common -import parsing_representation as pr +from jedi import debug +from jedi import common +from jedi import parsing_representation as pr class ParserError(Exception): diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index f429e92f..65210124 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -39,10 +39,10 @@ import os import re import tokenize -from _compatibility import next, literal_eval, cleandoc, Python3Method, \ +from jedi._compatibility import next, literal_eval, cleandoc, Python3Method, \ encoding, property, unicode, is_py3k -import common -import debug +from jedi import common +from jedi import debug class Base(object): diff --git a/jedi/recursion.py b/jedi/recursion.py index e12a7ce4..cf623e61 100644 --- a/jedi/recursion.py +++ b/jedi/recursion.py @@ -7,11 +7,11 @@ Next to :mod:`cache` this module also makes |jedi| not thread-safe. Why? ``ExecutionRecursionDecorator`` uses class variables to count the function calls. """ -import parsing_representation as pr -import evaluate_representation as er -import debug +from jedi import parsing_representation as pr +from jedi import evaluate_representation as er +from jedi import debug +from jedi import settings import builtin -import settings class RecursionDecorator(object): diff --git a/jedi/refactoring.py b/jedi/refactoring.py index f478d4e0..0f3eeade 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -15,10 +15,10 @@ following functions (sometimes bug-prone): from __future__ import with_statement -import modules -import difflib -import helpers -import parsing_representation as pr +from jedi import modules +from jedi import difflib +from jedi import helpers +from jedi import parsing_representation as pr class Refactoring(object): From 7b6a4c9a2b6b798bed370351584889952cafe8d6 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 15 Mar 2013 16:01:23 +0430 Subject: [PATCH 055/109] some corrections of imports --- jedi/api.py | 6 +++--- jedi/api_classes.py | 4 ++-- jedi/dynamic.py | 2 +- jedi/evaluate.py | 4 ++-- jedi/evaluate_representation.py | 2 +- jedi/imports.py | 2 +- jedi/recursion.py | 2 +- jedi/refactoring.py | 3 ++- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 429ec603..51c1071c 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -20,14 +20,14 @@ from jedi import settings from jedi import helpers from jedi import common from jedi import cache -from jedi import api_classes from jedi import modules -from jedi import keywords from jedi._compatibility import next, unicode +import evaluate +import keywords +import api_classes import evaluate_representation as er import dynamic import imports -import evaluate import builtin diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 25a4cf48..df63198d 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -10,11 +10,11 @@ import warnings import functools from jedi._compatibility import unicode, next -from jedi import recursion from jedi import settings from jedi import parsing_representation as pr -from jedi import keywords from jedi import cache +import keywords +import recursion import dynamic import evaluate import imports diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 687c46da..8cc0b3eb 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -60,8 +60,8 @@ from jedi import parsing_representation as pr from jedi import modules from jedi import settings from jedi import debug -from jedi import api_classes from jedi import fast_parser +import api_classes import evaluate import imports import evaluate_representation as er diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 016e7075..551852d7 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -77,9 +77,9 @@ from jedi import common from jedi import cache from jedi import parsing_representation as pr from jedi import debug -from jedi import recursion -import docstrings import evaluate_representation as er +import recursion +import docstrings import builtin import imports import dynamic diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 31fba868..99fded99 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -18,7 +18,7 @@ from jedi import cache from jedi import helpers from jedi import debug from jedi import common -from jedi import recursion +import recursion import docstrings import imports import evaluate diff --git a/jedi/imports.py b/jedi/imports.py index 90c119cd..72a3d0f6 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -19,11 +19,11 @@ import os import pkgutil import imp import sys +import itertools from jedi import modules from jedi import debug from jedi import parsing_representation as pr -from jedi import itertools from jedi import cache import builtin import evaluate diff --git a/jedi/recursion.py b/jedi/recursion.py index cf623e61..1727897a 100644 --- a/jedi/recursion.py +++ b/jedi/recursion.py @@ -8,9 +8,9 @@ Next to :mod:`cache` this module also makes |jedi| not thread-safe. Why? calls. """ from jedi import parsing_representation as pr -from jedi import evaluate_representation as er from jedi import debug from jedi import settings +import evaluate_representation as er import builtin diff --git a/jedi/refactoring.py b/jedi/refactoring.py index 0f3eeade..a724ec16 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -15,8 +15,9 @@ following functions (sometimes bug-prone): from __future__ import with_statement +import difflib + from jedi import modules -from jedi import difflib from jedi import helpers from jedi import parsing_representation as pr From 8c51d1e53e06118653b5905ab23576055a8f9130 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 15 Mar 2013 16:31:35 +0430 Subject: [PATCH 056/109] increase speed test limit a little bit, because it fails sometimes on travis --- test/test_regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_regression.py b/test/test_regression.py index 2e0da3cf..6c4181f8 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -514,7 +514,7 @@ class TestSpeed(TestBase): return wrapper return decorated - @_check_speed(0.1) + @_check_speed(0.2) def test_os_path_join(self): s = "from posixpath import join; join('', '')." assert len(self.complete(s)) > 10 # is a str completion From f70e425c4a0355ce6114f2f27944ecc2f59f8b72 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 13:08:55 +0100 Subject: [PATCH 057/109] Lowercase the str returned by BaseDefinition.type --- jedi/api_classes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 2be10641..88300d9e 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -120,20 +120,20 @@ class BaseDefinition(object): Finally, here is what you can get from :attr:`type`: >>> defs[0].type - 'Module' + 'module' >>> defs[1].type - 'Class' + 'class' >>> defs[2].type - 'Instance' + 'instance' >>> defs[3].type - 'Function' + 'function' """ # generate the type stripped = self.definition if isinstance(self.definition, er.InstanceElement): stripped = self.definition.var - return type(stripped).__name__ + return type(stripped).__name__.lower() @property def path(self): From 3caebcb5aa3ff6986262df69b1564d348d29aa80 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 13:39:48 +0100 Subject: [PATCH 058/109] Add test_basedefinition_type --- test/test_api_classes.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 test/test_api_classes.py diff --git a/test/test_api_classes.py b/test/test_api_classes.py new file mode 100644 index 00000000..3856d6be --- /dev/null +++ b/test/test_api_classes.py @@ -0,0 +1,23 @@ +import pytest + +from jedi import api + + +def make_definitions(): + return api.defined_names(""" + import sys + + class C: + pass + + x = C() + + def f(): + pass + """) + + +@pytest.mark.parametrize('definition', make_definitions()) +def test_basedefinition_type(definition): + assert definition.type in ('module', 'class', 'instance', 'function', + 'statement', 'import') From f791e96d9da075ca7b69fd7218c0200a0e0f1827 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 13:47:31 +0100 Subject: [PATCH 059/109] Fix failing test_basedefinition_type --- jedi/api_classes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 88300d9e..2baff8c3 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -133,6 +133,8 @@ class BaseDefinition(object): stripped = self.definition if isinstance(self.definition, er.InstanceElement): stripped = self.definition.var + if isinstance(stripped, pr.Name): + stripped = stripped.parent return type(stripped).__name__.lower() @property From 956ad502766eddbaf3c81672a30e58c814ba8437 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 14:10:24 +0100 Subject: [PATCH 060/109] Make more examples in make_definitions --- test/test_api_classes.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test/test_api_classes.py b/test/test_api_classes.py index 3856d6be..8e4f76cf 100644 --- a/test/test_api_classes.py +++ b/test/test_api_classes.py @@ -1,10 +1,17 @@ +import textwrap + import pytest from jedi import api def make_definitions(): - return api.defined_names(""" + """ + Return a list of definitions for parametrized tests. + + :rtype: [jedi.api_classes.BaseDefinition] + """ + source = textwrap.dedent(""" import sys class C: @@ -16,6 +23,20 @@ def make_definitions(): pass """) + definitions = [] + definitions += api.defined_names(source) + + source += textwrap.dedent(""" + variable = sys or C or x or f""") + lines = source.splitlines() + script = api.Script(source, len(lines), len('variable'), None) + definitions += script.definition() + + script2 = api.Script(source, 4, len('class C'), None) + definitions += script2.related_names() + + return definitions + @pytest.mark.parametrize('definition', make_definitions()) def test_basedefinition_type(definition): From a0c796087aff5055e0e4cef7e5b3ea46ce1550e2 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 14:24:42 +0100 Subject: [PATCH 061/109] Generate "param" definition in make_definitions --- test/test_api_classes.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test_api_classes.py b/test/test_api_classes.py index 8e4f76cf..69bbf2b3 100644 --- a/test/test_api_classes.py +++ b/test/test_api_classes.py @@ -35,10 +35,14 @@ def make_definitions(): script2 = api.Script(source, 4, len('class C'), None) definitions += script2.related_names() + source_param = "def f(a): return a" + script_param = api.Script(source_param, 1, len(source_param), None) + definitions += script_param.goto() + return definitions @pytest.mark.parametrize('definition', make_definitions()) def test_basedefinition_type(definition): assert definition.type in ('module', 'class', 'instance', 'function', - 'statement', 'import') + 'statement', 'import', 'param') From c8c1113c55659c72603247452ec6be6767a5b319 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 14:26:58 +0100 Subject: [PATCH 062/109] Test generator and lambda --- test/test_api_classes.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test/test_api_classes.py b/test/test_api_classes.py index 69bbf2b3..bcb7a722 100644 --- a/test/test_api_classes.py +++ b/test/test_api_classes.py @@ -21,13 +21,18 @@ def make_definitions(): def f(): pass + + def g(): + yield + + h = lambda: None """) definitions = [] definitions += api.defined_names(source) source += textwrap.dedent(""" - variable = sys or C or x or f""") + variable = sys or C or x or f or g or h""") lines = source.splitlines() script = api.Script(source, len(lines), len('variable'), None) definitions += script.definition() From be33f0ad98fdf18153653917328ee006b5b4d9e8 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 14:39:51 +0100 Subject: [PATCH 063/109] Fix previously failing test_follow_definition --- test/test_regression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_regression.py b/test/test_regression.py index 2e0da3cf..a6e210d0 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -319,7 +319,7 @@ class TestRegression(TestBase): # attributes objs = itertools.chain.from_iterable(r.follow_definition() for r in c) types = [o.type for o in objs] - assert 'Import' not in types and 'Class' in types + assert 'import' not in types and 'class' in types def test_keyword_definition_doc(self): """ github jedi-vim issue #44 """ From df08122639f85d3e0e3067c61753172b37f24bd4 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 14:51:59 +0100 Subject: [PATCH 064/109] Fix a failure when run with Python 3 --- jedi/api.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/jedi/api.py b/jedi/api.py index 86b9a369..87e62c56 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -272,7 +272,7 @@ class Script(object): d = set([api_classes.Definition(s) for s in scopes if not isinstance(s, imports.ImportPath._GlobalNamespace)]) - return sorted(d, key=lambda x: (x.module_path, x.start_pos)) + return self._sorted_defs(d) @api_classes._clear_caches_after_call def goto(self): @@ -285,7 +285,7 @@ class Script(object): :rtype: list of :class:`api_classes.Definition` """ d = [api_classes.Definition(d) for d in set(self._goto()[0])] - return sorted(d, key=lambda x: (x.module_path, x.start_pos)) + return self._sorted_defs(d) def _goto(self, add_import_name=False): """ @@ -371,7 +371,7 @@ class Script(object): else: names.append(api_classes.RelatedName(d.names[-1], d)) - return sorted(set(names), key=lambda x: (x.module_path, x.start_pos)) + return self._sorted_defs(set(names)) def get_in_function_call(self): """ @@ -492,6 +492,12 @@ class Script(object): match = re.match(r'^(.*?)(\.|)(\w?[\w\d]*)$', path, flags=re.S) return match.groups() + @staticmethod + def _sorted_defs(d): + # Note: `or ''` below is required because `module_path` could be + # None and you can't compare None and str in Python 3. + return sorted(d, key=lambda x: (x.module_path or '', x.start_pos)) + def defined_names(source, source_path=None, source_encoding='utf-8'): """ From 8be6b64bc79e306b70f49fda6dd3c10545d88664 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 15 Mar 2013 21:01:44 +0430 Subject: [PATCH 065/109] add @tkf to main authors, concrats man - you earned it - more than enough. --- AUTHORS.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index 93574a5f..a603ab28 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -2,14 +2,13 @@ Main Authors ============ David Halter (@davidhalter) - +Takafumi Arakaki (@tkf) Code Contributors ================= Danilo Bargen (@dbrgn) tek (@tek) -Takafumi Arakaki (@tkf) Yasha Borevich (@jjay) Aaron Griffin andviro (@andviro) From 07ba6658dd3ecd539f344ad5cc7ad17c53ab5eb0 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 16 Mar 2013 20:37:12 +0100 Subject: [PATCH 066/109] Add test_no_duplicate_modules --- test/test_regression.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/test_regression.py b/test/test_regression.py index d71dec1a..201ca063 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -537,5 +537,41 @@ def test_settings_module(): assert cache.settings is settings +def test_no_duplicate_modules(): + """ + Make sure that import hack works as expected. + + Jedi does an import hack (see: jedi/__init__.py) to have submodules + with circular dependencies. The modules in this circular dependency + "loop" must be imported by ``import `` rather than normal + ``from jedi import `` (or ``from . jedi ...``). This test + make sure that this is satisfied. + + See also: + + - `#160 `_ + - `#161 `_ + """ + import sys + jedipath = os.path.dirname(os.path.abspath(jedi.__file__)) + + def is_submodule(m): + try: + filepath = m.__file__ + except AttributeError: + return False + return os.path.abspath(filepath).startswith(jedipath) + + modules = list(filter(is_submodule, sys.modules.values())) + top_modules = [m for m in modules if not m.__name__.startswith('jedi.')] + for m in modules: + for tm in top_modules: + try: + imported = getattr(m, tm.__name__) + except AttributeError: + continue + assert imported is tm + + if __name__ == '__main__': unittest.main() From f5fee5f0df1b7f869efe10cf95e138aae5dd1e46 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 16 Mar 2013 21:03:34 +0100 Subject: [PATCH 067/109] Fix wrong imports - "from _compatibility import ..." (not in circular imports) - "from jedi import builtin" (one of circular imports) - "api_classes = api.api_classes" ("from jedi import api_classes" is not supported) --- jedi/evaluate.py | 2 +- jedi/evaluate_representation.py | 2 +- jedi/keywords.py | 2 +- jedi/modules.py | 3 +-- jedi/parsing.py | 2 +- test/test_regression.py | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/jedi/evaluate.py b/jedi/evaluate.py index 551852d7..c76bbb27 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -68,11 +68,11 @@ backtracking algorithm. .. todo:: nonlocal statement, needed or can be ignored? (py3k) """ -from _compatibility import next, hasattr, is_py3k, unicode, reraise import sys import itertools +from jedi._compatibility import next, hasattr, is_py3k, unicode, reraise from jedi import common from jedi import cache from jedi import parsing_representation as pr diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 99fded99..f5c6e991 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -12,7 +12,7 @@ they change classes in Python 3. import copy import itertools -from _compatibility import property, use_metaclass, next, hasattr +from jedi._compatibility import property, use_metaclass, next, hasattr from jedi import parsing_representation as pr from jedi import cache from jedi import helpers diff --git a/jedi/keywords.py b/jedi/keywords.py index 07461bb7..5495ae32 100644 --- a/jedi/keywords.py +++ b/jedi/keywords.py @@ -2,7 +2,7 @@ import pydoc import keyword from jedi._compatibility import is_py3k -from jedi import builtin +import builtin try: from pydoc_data import topics as pydoc_topics diff --git a/jedi/modules.py b/jedi/modules.py index b6c8f1e8..9277e742 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -15,13 +15,12 @@ Apart from those classes there's a ``sys.path`` fetching function, as well as """ from __future__ import with_statement -from _compatibility import exec_function, unicode, is_py25, literal_eval - import re import tokenize import sys import os +from jedi._compatibility import exec_function, unicode, is_py25, literal_eval from jedi import cache from jedi import parsing from jedi import parsing_representation as pr diff --git a/jedi/parsing.py b/jedi/parsing.py index 17ca7d49..69607fec 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -15,11 +15,11 @@ within the statement. This lowers memory usage and cpu time and reduces the complexity of the ``Parser`` (there's another parser sitting inside ``Statement``, which produces ``Array`` and ``Call``). """ -from _compatibility import next, StringIO import tokenize import keyword +from jedi._compatibility import next, StringIO from jedi import debug from jedi import common from jedi import parsing_representation as pr diff --git a/test/test_regression.py b/test/test_regression.py index 201ca063..8c8209dc 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -18,7 +18,7 @@ from .base import TestBase, unittest, cwd_at import jedi from jedi._compatibility import is_py25, utf8, unicode from jedi import api -from jedi import api_classes +api_classes = api.api_classes #jedi.set_debug_function(jedi.debug.print_to_stdout) From 4aac06eb866df930cbb32c05048b465fd500b51b Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 13 Mar 2013 16:33:25 +0100 Subject: [PATCH 068/109] Remove tags for known failures due to the import hack --- test/test_cache.py | 3 --- test/test_regression.py | 3 --- tox.ini | 6 ------ 3 files changed, 12 deletions(-) diff --git a/test/test_cache.py b/test/test_cache.py index 8352fd11..9f845fc2 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -1,5 +1,3 @@ -import pytest - from jedi import settings from jedi.cache import ParserCacheItem, _ModulePickling @@ -7,7 +5,6 @@ from jedi.cache import ParserCacheItem, _ModulePickling ModulePickling = _ModulePickling() -@pytest.mark.skipif("sys.version_info >= (3,0)") def test_modulepickling_change_cache_dir(monkeypatch, tmpdir): """ ModulePickling should not save old cache when cache_directory is changed. diff --git a/test/test_regression.py b/test/test_regression.py index d71dec1a..6ed6a9f9 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -11,8 +11,6 @@ import itertools import os import textwrap -import pytest - from .base import TestBase, unittest, cwd_at import jedi @@ -527,7 +525,6 @@ class TestSpeed(TestBase): #print(jedi.imports.imports_processed) -@pytest.mark.skipif("sys.version_info >= (3,0)") def test_settings_module(): """ jedi.settings and jedi.cache.settings must be the same module. diff --git a/tox.ini b/tox.ini index 6b2bad01..6dc16930 100644 --- a/tox.ini +++ b/tox.ini @@ -14,9 +14,3 @@ deps = deps = unittest2 {[testenv]deps} -[testenv:py32] -# TODO: Without this setting, test uses ~/.cache/jedi/. -# There could be a bug due to import hack. -# See test_settings_module in test/test_regression.py. -setenv = - XDG_CACHE_HOME={envtmpdir}/cache From febc65be6fa144723261a08d3d864abf47cc15fd Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 16 Mar 2013 21:07:02 +0100 Subject: [PATCH 069/109] Skip test_no_duplicate_modules for jedi.__init__ --- test/test_regression.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test_regression.py b/test/test_regression.py index 8c8209dc..e6027c9f 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -565,6 +565,10 @@ def test_no_duplicate_modules(): modules = list(filter(is_submodule, sys.modules.values())) top_modules = [m for m in modules if not m.__name__.startswith('jedi.')] for m in modules: + if m is jedi: + # py.test automatically improts `jedi.*` when --doctest-modules + # is given. So this test cannot succeeds. + continue for tm in top_modules: try: imported = getattr(m, tm.__name__) From c25ee531a259a79cf919f5c4553fab315728987d Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Fri, 15 Mar 2013 14:36:12 +0100 Subject: [PATCH 070/109] AttributeError: 'Generator' object has no attribute 'start_pos' Running py.test raises this error: ```tb test/test_api_classes.py:50: in > @pytest.mark.parametrize('definition', make_definitions()) test/test_api_classes.py:38: in make_definitions > definitions += script.definition() jedi/api_classes.py:44: in wrapper > result = func(*args, **kwds) jedi/api.py:274: in definition > if not isinstance(s, imports.ImportPath._GlobalNamespace)]) jedi/api_classes.py:418: in __init__ > super(Definition, self).__init__(definition, definition.start_pos) E AttributeError: 'Generator' object has no attribute 'start_pos' ``` --- test/test_api_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_api_classes.py b/test/test_api_classes.py index bcb7a722..747eabdc 100644 --- a/test/test_api_classes.py +++ b/test/test_api_classes.py @@ -32,7 +32,7 @@ def make_definitions(): definitions += api.defined_names(source) source += textwrap.dedent(""" - variable = sys or C or x or f or g or h""") + variable = sys or C or x or f or g or g() or h""") lines = source.splitlines() script = api.Script(source, len(lines), len('variable'), None) definitions += script.definition() From 6a2e535bcbb20d8331ba7bab749c03725aea7f25 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 16 Mar 2013 21:49:46 +0100 Subject: [PATCH 071/109] Add er.Generator.__getattr__ --- jedi/evaluate_representation.py | 3 +++ test/test_api_classes.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 99fded99..019e038b 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -773,6 +773,9 @@ class Generator(use_metaclass(cache.CachedMetaClass, pr.Base)): def parent(self): return self.func.parent + def __getattr__(self, name): + return getattr(self.func, name) + def __repr__(self): return "<%s of %s>" % (type(self).__name__, self.func) diff --git a/test/test_api_classes.py b/test/test_api_classes.py index 747eabdc..2da3e13e 100644 --- a/test/test_api_classes.py +++ b/test/test_api_classes.py @@ -50,4 +50,4 @@ def make_definitions(): @pytest.mark.parametrize('definition', make_definitions()) def test_basedefinition_type(definition): assert definition.type in ('module', 'class', 'instance', 'function', - 'statement', 'import', 'param') + 'generator', 'statement', 'import', 'param') From 2cc6edfa7c0e1625486723d35dd072697f18f731 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sat, 16 Mar 2013 22:57:31 +0100 Subject: [PATCH 072/109] Fix Completion.follow_definition --- jedi/api_classes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index eb6b58d7..842fe195 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -393,7 +393,7 @@ class Completion(BaseDefinition): """ if self._followed_definitions is None: if self.definition.isinstance(pr.Statement): - defs = er.follow_statement(self.definition) + defs = evaluate.follow_statement(self.definition) elif self.definition.isinstance(pr.Import): defs = imports.strip_imports([self.definition]) else: From d1150cfdae2fc7b5d302cad0fcc3738fe2df7c5d Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Sun, 17 Mar 2013 13:56:54 +0100 Subject: [PATCH 073/109] List allowed attributes in Generator.__getattr__ --- jedi/evaluate_representation.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 019e038b..f9bbd2c9 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -769,11 +769,12 @@ class Generator(use_metaclass(cache.CachedMetaClass, pr.Base)): debug.warning('Tried to get array access on a generator', self) return [] - @property - def parent(self): - return self.func.parent - def __getattr__(self, name): + if name not in ['start_pos', 'end_pos', 'parent', 'get_imports', + 'asserts', 'doc', 'docstr', 'get_parent_until', 'get_code', + 'subscopes']: + raise AttributeError("Accessing %s of %s is not allowed." + % (self, name)) return getattr(self.func, name) def __repr__(self): From 09f39d891abab7e92bcd9c1e2442aa001d433278 Mon Sep 17 00:00:00 2001 From: David Halter Date: Mon, 18 Mar 2013 08:25:14 +0430 Subject: [PATCH 074/109] remove __all__ from api, because not needed, fixes #180 --- jedi/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/jedi/api.py b/jedi/api.py index 1ed062d3..af94e287 100644 --- a/jedi/api.py +++ b/jedi/api.py @@ -7,7 +7,6 @@ catch :exc:`NotFoundError` which is being raised if your completion is not possible. """ from __future__ import with_statement -__all__ = ['Script', 'NotFoundError', 'set_debug_function', '_quick_complete'] import re import os From 8734e0efe2d7b6eff1dea1d3ca28651fec861517 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 19 Mar 2013 20:33:15 +0100 Subject: [PATCH 075/109] Use cPickle if available --- jedi/cache.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jedi/cache.py b/jedi/cache.py index 84d9ede7..4a729dc5 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -21,7 +21,10 @@ from __future__ import with_statement import time import os import sys -import pickle +try: + import cPickle as pickle +except: + import pickle from _compatibility import json import settings From f8d3339f2fdf9259cf263c35c443e987523fdd31 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 19 Mar 2013 21:32:01 +0100 Subject: [PATCH 076/109] Remove old version of FS cache automatically --- jedi/cache.py | 30 ++++++++++++++++++++++++++++-- test/conftest.py | 12 ++++++++++++ test/test_cache.py | 28 ++++++++++++++++++++++++++-- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/jedi/cache.py b/jedi/cache.py index e04b40e3..67fc84c5 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -25,6 +25,7 @@ try: import cPickle as pickle except: import pickle +import shutil from jedi._compatibility import json from jedi import settings @@ -219,6 +220,20 @@ def save_module(path, name, parser, pickling=True): class _ModulePickling(object): + + version = 1 + """ + Version number (integer) for file system cache. + + Increment this number when there are any incompatible changes in + parser representation classes. For example, the following changes + are regarded as incompatible. + + - Class name is changed. + - Class is moved to another module. + - Defined slot of the class is changed. + """ + def __init__(self): self.__index = None self.py_version = '%s.%s' % sys.version_info[:2] @@ -259,9 +274,16 @@ class _ModulePickling(object): if self.__index is None: try: with open(self._get_path('index.json')) as f: - self.__index = json.load(f) + data = json.load(f) except IOError: self.__index = {} + else: + # 0 means version is not defined (= always delete cache): + if data.get('version', 0) < self.version: + self.delete_cache() + self.__index = {} + else: + self.__index = data['index'] return self.__index def _remove_old_modules(self): @@ -272,10 +294,14 @@ class _ModulePickling(object): self._index # reload index def _flush_index(self): + data = {'version': self.version, 'index': self._index} with open(self._get_path('index.json'), 'w') as f: - json.dump(self._index, f) + json.dump(data, f) self.__index = None + def delete_cache(self): + shutil.rmtree(settings.cache_directory) + def _get_hashed_path(self, path): return self._get_path('%s_%s.pkl' % (self.py_version, hash(path))) diff --git a/test/conftest.py b/test/conftest.py index c5a00879..12a01b13 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -65,6 +65,18 @@ def pytest_generate_tests(metafunc): refactor.collect_dir_tests(base_dir, test_files)) +@pytest.fixture() +def isolated_jedi_cache(monkeypatch, tmpdir): + """ + Set `jedi.settings.cache_directory` to a temporary directory during test. + + Same as `clean_jedi_cache`, but create the temporary directory for + each test case (scope='function'). + """ + settings = base.jedi.settings + monkeypatch.setattr(settings, 'cache_directory', str(tmpdir)) + + @pytest.fixture(scope='session') def clean_jedi_cache(request): """ diff --git a/test/test_cache.py b/test/test_cache.py index 9f845fc2..c27dc705 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -1,3 +1,5 @@ +import pytest + from jedi import settings from jedi.cache import ParserCacheItem, _ModulePickling @@ -21,10 +23,32 @@ def test_modulepickling_change_cache_dir(monkeypatch, tmpdir): monkeypatch.setattr(settings, 'cache_directory', dir_1) ModulePickling.save_module(path_1, item_1) - cached = ModulePickling.load_module(path_1, item_1.change_time - 1) + cached = load_stored_item(ModulePickling, path_1, item_1) assert cached == item_1.parser monkeypatch.setattr(settings, 'cache_directory', dir_2) ModulePickling.save_module(path_2, item_2) - cached = ModulePickling.load_module(path_1, item_1.change_time - 1) + cached = load_stored_item(ModulePickling, path_1, item_1) assert cached is None + + +def load_stored_item(cache, path, item): + """Load `item` stored at `path` in `cache`.""" + return cache.load_module(path, item.change_time - 1) + + +@pytest.mark.usefixtures("isolated_jedi_cache") +def test_modulepickling_delete_incompatible_cache(): + item = ParserCacheItem('fake parser') + path = 'fake path' + + cache1 = _ModulePickling() + cache1.version = 1 + cache1.save_module(path, item) + cached1 = load_stored_item(cache1, path, item) + assert cached1 == item.parser + + cache2 = _ModulePickling() + cache2.version = 2 + cached2 = load_stored_item(cache2, path, item) + assert cached2 is None From 942b66ba5bf0abaefb9a602a61e46c9e9f337ab9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 19 Mar 2013 21:55:57 +0100 Subject: [PATCH 077/109] Separate directory for each Python version --- jedi/cache.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/jedi/cache.py b/jedi/cache.py index 67fc84c5..933e38af 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -240,7 +240,7 @@ class _ModulePickling(object): def load_module(self, path, original_changed_time): try: - pickle_changed_time = self._index[self.py_version][path] + pickle_changed_time = self._index[path] except KeyError: return None if original_changed_time is not None \ @@ -258,10 +258,10 @@ class _ModulePickling(object): def save_module(self, path, parser_cache_item): self.__index = None try: - files = self._index[self.py_version] + files = self._index except KeyError: files = {} - self._index[self.py_version] = files + self._index = files with open(self._get_hashed_path(path), 'wb') as f: pickle.dump(parser_cache_item, f, pickle.HIGHEST_PROTOCOL) @@ -300,16 +300,19 @@ class _ModulePickling(object): self.__index = None def delete_cache(self): - shutil.rmtree(settings.cache_directory) + shutil.rmtree(self._cache_directory()) def _get_hashed_path(self, path): - return self._get_path('%s_%s.pkl' % (self.py_version, hash(path))) + return self._get_path('%s.pkl' % hash(path)) def _get_path(self, file): - dir = settings.cache_directory + dir = self._cache_directory() if not os.path.exists(dir): os.makedirs(dir) - return dir + os.path.sep + file + return os.path.join(dir, file) + + def _cache_directory(self): + return os.path.join(settings.cache_directory, self.py_version) # is a singleton From 27d007adba2315b19364fdc38a05272cbaad8f0f Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 19 Mar 2013 21:56:40 +0100 Subject: [PATCH 078/109] Compare version using "!=" Cache is not forward compatible. --- jedi/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/cache.py b/jedi/cache.py index 933e38af..2b744337 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -279,7 +279,7 @@ class _ModulePickling(object): self.__index = {} else: # 0 means version is not defined (= always delete cache): - if data.get('version', 0) < self.version: + if data.get('version', 0) != self.version: self.delete_cache() self.__index = {} else: From aca1cbfa4b9dc428370415138a8b4945c4a3f3a1 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 20 Mar 2013 16:44:17 +0100 Subject: [PATCH 079/109] Use "jedi/cpython-27" instead of "jedi/2.7" for FS cache --- jedi/cache.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/jedi/cache.py b/jedi/cache.py index 2b744337..20805845 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -236,7 +236,16 @@ class _ModulePickling(object): def __init__(self): self.__index = None - self.py_version = '%s.%s' % sys.version_info[:2] + self.py_tag = 'cpython-%s%s' % sys.version_info[:2] + """ + Short name for distinguish Python implementations and versions. + + It's like `sys.implementation.cache_tag` but for Python < 3.3 + we generate something similar. See: + http://docs.python.org/3/library/sys.html#sys.implementation + + .. todo:: Detect interpreter (e.g., PyPy). + """ def load_module(self, path, original_changed_time): try: @@ -312,7 +321,7 @@ class _ModulePickling(object): return os.path.join(dir, file) def _cache_directory(self): - return os.path.join(settings.cache_directory, self.py_version) + return os.path.join(settings.cache_directory, self.py_tag) # is a singleton From d481a7aae49649d37d00fb2c85f161810f5d0fbc Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Fri, 22 Mar 2013 13:39:07 +0000 Subject: [PATCH 080/109] Created find_module helper to handle compatibility with python 3.3 Moved package checking logic in follow_str function Created find_module compatibility helper method Conditional implementation of load_module for python 3.3 --- jedi/_compatibility.py | 43 +++++++++++++++++++++++++++++++++++++++++- jedi/imports.py | 20 ++++++++++++++------ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 9a704304..454511dd 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -7,11 +7,52 @@ Most of the code here is necessary to support Python 2.5. Once this dependency will be dropped, we'll get rid of most code. """ import sys +import imp +import os +import io +try: + import importlib +except: + pass is_py3k = sys.hexversion >= 0x03000000 - +is_py33 = sys.hexversion >= 0x03030000 is_py25 = sys.hexversion < 0x02060000 +if is_py33: + def find_module(string, path=None): + importing = None + if path is not None: + importing = importlib.find_loader(string, path) + else: + importing = importlib.find_loader(string) + + returning = (None, None, None) + try: + filename = importing.get_filename(string) + if filename and os.path.exists(filename): + returning = (open(filename, 'U'), filename, False) + else: + returning = (None, filename, False) + except AttributeError: + if importing is None: + returning = (None, None, False) + else: + returning = (None, importing.load_module(string).__name__, False) + + return returning +else: + def find_module(string, path=None): + importing = None + if path is None: + importing = imp.find_module(string) + else: + importing = imp.find_module(string, path) + + returning = (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) + + return returning + # next was defined in python 2.6, in python 3 obj.next won't be possible # anymore try: diff --git a/jedi/imports.py b/jedi/imports.py index 72a3d0f6..a006f3e7 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -21,6 +21,7 @@ import imp import sys import itertools +from jedi._compatibility import find_module from jedi import modules from jedi import debug from jedi import parsing_representation as pr @@ -238,20 +239,22 @@ class ImportPath(pr.Base): global imports_processed imports_processed += 1 + importing = None if path is not None: - return imp.find_module(string, [path]) + importing = find_module(string, [path]) else: debug.dbg('search_module', string, self.file_path) # Override the sys.path. It works only good that way. # Injecting the path directly into `find_module` did not work. sys.path, temp = sys_path_mod, sys.path try: - i = imp.find_module(string) + importing = find_module(string) except ImportError: sys.path = temp raise sys.path = temp - return i + + return importing if self.file_path: sys_path_mod = list(self.sys_path_with_modifications()) @@ -259,6 +262,9 @@ class ImportPath(pr.Base): else: sys_path_mod = list(modules.get_sys_path()) + def module_not_found(): + raise ModuleNotFound('The module you searched has not been found') + current_namespace = (None, None, None) # now execute those paths rest = [] @@ -277,12 +283,14 @@ class ImportPath(pr.Base): if current_namespace[1]: rest = self.import_path[i:] else: - raise ModuleNotFound( - 'The module you searched has not been found') + module_not_found() + + if current_namespace == (None, None, False): + module_not_found() sys_path_mod.pop(0) # TODO why is this here? path = current_namespace[1] - is_package_directory = current_namespace[2][2] == imp.PKG_DIRECTORY + is_package_directory = current_namespace[2] f = None if is_package_directory or current_namespace[0]: From de849fbe8a1e08650f9556404a1959249375d8d6 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Fri, 22 Mar 2013 19:11:44 +0000 Subject: [PATCH 081/109] Fixed follow_definition test --- jedi/_compatibility.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 454511dd..dff2e42d 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -24,6 +24,8 @@ if is_py33: importing = None if path is not None: importing = importlib.find_loader(string, path) + if importing is None: + raise ImportError else: importing = importlib.find_loader(string) @@ -39,7 +41,7 @@ if is_py33: returning = (None, None, False) else: returning = (None, importing.load_module(string).__name__, False) - + return returning else: def find_module(string, path=None): From be8ef33b695ed3a37972d9caff9a8e1c0c00bb61 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Sat, 23 Mar 2013 23:07:17 +0000 Subject: [PATCH 082/109] Fixed caching issues Fixed exception raised during cache reading Switched cache hashing to hashlib In python 3.3 the hash function is returning different hashes during different executions of the application. --- jedi/cache.py | 3 ++- jedi/parsing_representation.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/jedi/cache.py b/jedi/cache.py index 20805845..2b671700 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -21,6 +21,7 @@ from __future__ import with_statement import time import os import sys +import hashlib try: import cPickle as pickle except: @@ -312,7 +313,7 @@ class _ModulePickling(object): shutil.rmtree(self._cache_directory()) def _get_hashed_path(self, path): - return self._get_path('%s.pkl' % hash(path)) + return self._get_path('%s.pkl' % hashlib.md5(path.encode("utf-8")).hexdigest()) def _get_path(self, file): dir = self._cache_directory() diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 65210124..d1ea83ff 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -560,7 +560,7 @@ class Flow(Scope): @parent.setter def parent(self, value): self._parent = value - if self.next: + if hasattr(self, "next") and self.next: self.next.parent = value def get_code(self, first_indent=False, indention=' '): From 3ef564847d460c782de2cf2cccd16e9a8eba5567 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Sat, 23 Mar 2013 23:16:06 +0000 Subject: [PATCH 083/109] Simplified code for readability Splitted import compatibility function definition for better readability Simplified code for python 3.3 load_module implementation --- jedi/_compatibility.py | 73 ++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 31 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index dff2e42d..a3489869 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -19,41 +19,52 @@ is_py3k = sys.hexversion >= 0x03000000 is_py33 = sys.hexversion >= 0x03030000 is_py25 = sys.hexversion < 0x02060000 -if is_py33: - def find_module(string, path=None): - importing = None - if path is not None: - importing = importlib.find_loader(string, path) - if importing is None: - raise ImportError +def find_module_py33(string, path=None): + returning = (None, None, None) + importing = None + if path is not None: + importing = importlib.find_loader(string, path) + else: + importing = importlib.find_loader(string) + + if importing is None: + raise ImportError + + try: + filename = importing.get_filename(string) + if filename and os.path.exists(filename): + returning = (open(filename, 'U'), filename, False) else: - importing = importlib.find_loader(string) + returning = (None, filename, False) + except AttributeError: + returning = (None, importing.load_module(string).__name__, False) - returning = (None, None, None) - try: - filename = importing.get_filename(string) - if filename and os.path.exists(filename): - returning = (open(filename, 'U'), filename, False) - else: - returning = (None, filename, False) - except AttributeError: - if importing is None: - returning = (None, None, False) - else: - returning = (None, importing.load_module(string).__name__, False) - - return returning -else: - def find_module(string, path=None): - importing = None - if path is None: - importing = imp.find_module(string) - else: - importing = imp.find_module(string, path) + return returning - returning = (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) +def find_module_pre_py33(string, path=None): + importing = None + if path is None: + importing = imp.find_module(string) + else: + importing = imp.find_module(string, path) - return returning + return (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) + +def find_module(string, path=None): + """Provides information about a module. + + This function isolates the differences in importing libraries introduced with + python 3.3 on; it gets a module name and optionally a path. It will return a + tuple containin an open file for the module (if not builtin), the filename + or the name of the module if it is a builtin one and a boolean indicating + ig the module is contained in a package.""" + returning = (None, None, None) + if is_py33: + returning = find_module_py33(string, path) + else: + returning = find_module_pre_py33(string, path) + + return returning # next was defined in python 2.6, in python 3 obj.next won't be possible # anymore From 07ec134bc92736aafdf8b1203297431ca54eebd2 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Sun, 24 Mar 2013 10:14:14 +0000 Subject: [PATCH 084/109] Adding python 3.3 to test environment, mani fixes Added python 3.3 to test-suite Removed unused import Removed unused import Migrated to EAFP for attribute checking Bumped version of ModulePickling for migration to hashlib Added py33 environment to tox Fixed issue with package importing on python 3.3 --- .travis.yml | 1 + jedi/_compatibility.py | 12 +++++++----- jedi/cache.py | 2 +- jedi/imports.py | 6 ++---- jedi/parsing_representation.py | 4 +++- tox.ini | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index e1529542..a85a4d06 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ env: - TOXENV=py26 - TOXENV=py27 - TOXENV=py32 + - TOXENV=py33 install: - pip install --quiet --use-mirrors tox script: diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index a3489869..331c1f11 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -9,7 +9,6 @@ will be dropped, we'll get rid of most code. import sys import imp import os -import io try: import importlib except: @@ -31,11 +30,14 @@ def find_module_py33(string, path=None): raise ImportError try: - filename = importing.get_filename(string) - if filename and os.path.exists(filename): - returning = (open(filename, 'U'), filename, False) + if (importing.is_package(string)): + returning = (None, os.path.dirname(importing.path), True) else: - returning = (None, filename, False) + filename = importing.get_filename(string) + if filename and os.path.exists(filename): + returning = (open(filename, 'U'), filename, False) + else: + returning = (None, filename, False) except AttributeError: returning = (None, importing.load_module(string).__name__, False) diff --git a/jedi/cache.py b/jedi/cache.py index 2b671700..5d76f24e 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -222,7 +222,7 @@ def save_module(path, name, parser, pickling=True): class _ModulePickling(object): - version = 1 + version = 2 """ Version number (integer) for file system cache. diff --git a/jedi/imports.py b/jedi/imports.py index a006f3e7..db72925f 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -5,9 +5,8 @@ any actual importing done. This module is about finding modules in the filesystem. This can be quite tricky sometimes, because Python imports are not always that simple. -Currently the import process uses ``imp`` to find modules. In the future, it's -a goal to use ``importlib`` for this purpose. There's a `pull request -`_ for that. +This module uses imp for python up to 3.2 and importlib for python 3.3 on; the +correct implementation is delegated to _compatibility. This module also supports import autocompletion, which means to complete statements like ``from datetim`` (curser at the end would return ``datetime``). @@ -17,7 +16,6 @@ from __future__ import with_statement import os import pkgutil -import imp import sys import itertools diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index d1ea83ff..311f38ab 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -560,8 +560,10 @@ class Flow(Scope): @parent.setter def parent(self, value): self._parent = value - if hasattr(self, "next") and self.next: + try: self.next.parent = value + except AttributeError: + return def get_code(self, first_indent=False, indention=' '): stmts = [] diff --git a/tox.ini b/tox.ini index 6dc16930..8fa50614 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py25, py26, py27, py32 +envlist = py25, py26, py27, py32, py33 [testenv] deps = pytest From 0b67a08e48ee4c1430a3557f9bbccc150547aacc Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Mon, 25 Mar 2013 19:30:58 +0000 Subject: [PATCH 085/109] Using PathFinder rather than find_loader to correctly handle paths Using PathFinder rather than find_loader to correctly handle from ... import ... Moved away from find_loader in favour of PathFinder also when using sys.path --- jedi/_compatibility.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 331c1f11..e15ba463 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -22,9 +22,11 @@ def find_module_py33(string, path=None): returning = (None, None, None) importing = None if path is not None: - importing = importlib.find_loader(string, path) + importing = importlib.machinery.PathFinder.find_module(string, path) else: - importing = importlib.find_loader(string) + importing = importlib.machinery.PathFinder.find_module(string, sys.path) + if importing is None: + importing = importlib.find_loader(string) if importing is None: raise ImportError From 124595de6e503d5e840a43d893c1018190fd8413 Mon Sep 17 00:00:00 2001 From: Aldo Stracquadanio Date: Wed, 27 Mar 2013 00:19:39 +0000 Subject: [PATCH 086/109] Making it nicer Fixed typo in docstring and added some comments in find_module_py33 Removed a test that is not compatible with python 3.3 Better variable names in find_module implementation(s) Removed variable assignation in favor of direct return statement --- jedi/_compatibility.py | 48 +++++++++++++++++++++--------------------- test/completion/std.py | 9 -------- 2 files changed, 24 insertions(+), 33 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index e15ba463..29aef154 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -19,40 +19,43 @@ is_py33 = sys.hexversion >= 0x03030000 is_py25 = sys.hexversion < 0x02060000 def find_module_py33(string, path=None): - returning = (None, None, None) - importing = None + mod_info = (None, None, None) + loader = None if path is not None: - importing = importlib.machinery.PathFinder.find_module(string, path) + # Check for the module in the specidied path + loader = importlib.machinery.PathFinder.find_module(string, path) else: - importing = importlib.machinery.PathFinder.find_module(string, sys.path) - if importing is None: - importing = importlib.find_loader(string) + # Check for the module in sys.path + loader = importlib.machinery.PathFinder.find_module(string, sys.path) + if loader is None: + # Fallback to find builtins + loader = importlib.find_loader(string) - if importing is None: + if loader is None: raise ImportError try: - if (importing.is_package(string)): - returning = (None, os.path.dirname(importing.path), True) + if (loader.is_package(string)): + mod_info = (None, os.path.dirname(loader.path), True) else: - filename = importing.get_filename(string) + filename = loader.get_filename(string) if filename and os.path.exists(filename): - returning = (open(filename, 'U'), filename, False) + mod_info = (open(filename, 'U'), filename, False) else: - returning = (None, filename, False) + mod_info = (None, filename, False) except AttributeError: - returning = (None, importing.load_module(string).__name__, False) + mod_info = (None, loader.load_module(string).__name__, False) - return returning + return mod_info def find_module_pre_py33(string, path=None): - importing = None + mod_info = None if path is None: - importing = imp.find_module(string) + mod_info = imp.find_module(string) else: - importing = imp.find_module(string, path) + mod_info = imp.find_module(string, path) - return (importing[0], importing[1], importing[2][2] == imp.PKG_DIRECTORY) + return (mod_info[0], mod_info[1], mod_info[2][2] == imp.PKG_DIRECTORY) def find_module(string, path=None): """Provides information about a module. @@ -61,14 +64,11 @@ def find_module(string, path=None): python 3.3 on; it gets a module name and optionally a path. It will return a tuple containin an open file for the module (if not builtin), the filename or the name of the module if it is a builtin one and a boolean indicating - ig the module is contained in a package.""" - returning = (None, None, None) + if the module is contained in a package.""" if is_py33: - returning = find_module_py33(string, path) + return find_module_py33(string, path) else: - returning = find_module_pre_py33(string, path) - - return returning + return find_module_pre_py33(string, path) # next was defined in python 2.6, in python 3 obj.next won't be possible # anymore diff --git a/test/completion/std.py b/test/completion/std.py index 0eec6367..c4ed8c5a 100644 --- a/test/completion/std.py +++ b/test/completion/std.py @@ -89,12 +89,3 @@ def huhu(db): """ #? sqlite3.Connection() db - -# ----------------- -# various regression tests -# ----------------- - -#62 -import threading -#? ['_Verbose', '_VERBOSE'] -threading._Verbose From 492a6c5704cda72368af017ccf003ad12eb00428 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Fri, 5 Apr 2013 11:18:53 +0200 Subject: [PATCH 087/109] Added specific "Programming Language" trove classifiers (fixes #194) --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index bc9c1fb1..bb0ee4e9 100755 --- a/setup.py +++ b/setup.py @@ -31,7 +31,13 @@ setup(name='jedi', 'Intended Audience :: Developers', 'License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)', 'Operating System :: OS Independent', - 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.5', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.2', + 'Programming Language :: Python :: 3.3', 'Topic :: Software Development :: Libraries :: Python Modules', 'Topic :: Text Editors :: Integrated Development Environments (IDE)', 'Topic :: Utilities', From 327a1ded177650330e0a581e71cc1100bc3779a2 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 10 Apr 2013 15:47:44 +0200 Subject: [PATCH 088/109] Note about testing in README --- README.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.rst b/README.rst index dc32f115..b00ea40f 100644 --- a/README.rst +++ b/README.rst @@ -86,3 +86,15 @@ API for IDEs It's very easy to create an editor plugin that uses Jedi. See https://jedi.readthedocs.org/en/latest/docs/plugin-api.html for more information. + + +Testing +======= + +To run the tests:: + + pip install tox pytest + PIP_INSECURE=t tox + +Tests are also run automatically on `Travis CI +`_. From be7cecef009cb33adcf493c175d690dad45ead05 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 10 Apr 2013 16:04:56 +0200 Subject: [PATCH 089/109] Typos in docs --- docs/docs/history.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docs/history.rst b/docs/docs/history.rst index df2ecc69..227cd47d 100644 --- a/docs/docs/history.rst +++ b/docs/docs/history.rst @@ -5,7 +5,7 @@ A little history The Star Wars Jedi are awesome. My Jedi software tries to imitate a little bit of the precognition the Jedi have. There's even an awesome `scene -`_ of Monty Python Jedi's :-). +`_ of Monty Python Jedis :-). But actually the name hasn't so much to do with Star Wars. It's part of my second name. @@ -13,13 +13,13 @@ second name. After I explained Guido van Rossum, how some parts of my auto-completion work, he said (we drank a beer or two): - *Oh, that worries me* + *"Oh, that worries me..."* When it's finished, I hope he'll like it :-) -I actually started Jedi, because there were no good solutions available for -VIM. Most auto-completions just didn't work well. The only good solution was -PyCharm. I just like my good old VIM. Rope was never really intended to be an +I actually started Jedi, because there were no good solutions available for VIM. +Most auto-completions just didn't work well. The only good solution was PyCharm. +But I like my good old VIM. Rope was never really intended to be an auto-completion (and also I really hate project folders for my Python scripts). It's more of a refactoring suite. So I decided to do my own version of a completion, which would execute non-dangerous code. But I soon realized, that From 798136787f57a39fdb1f78b299a5e15f743422bf Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 10 Apr 2013 16:30:32 +0200 Subject: [PATCH 090/109] Configuration to run coveralls on py33 --- .gitignore | 4 +++- .travis.yml | 3 ++- tox.ini | 7 ++++++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index ad9b77e4..4755f455 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,11 @@ *~ *.swp *.swo +*.pyc .ropeproject .tox -*.pyc +.coveralls.yml +.coverage /build/ /docs/_build/ /dist/ diff --git a/.travis.yml b/.travis.yml index a85a4d06..8c29ecb1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,7 @@ env: - TOXENV=py32 - TOXENV=py33 install: - - pip install --quiet --use-mirrors tox + - pip install --quiet --use-mirrors tox coveralls script: - tox + - if [ $TOXENV == "py33" ]; then coveralls; fi diff --git a/tox.ini b/tox.ini index 8fa50614..c6bf7e98 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = py25, py26, py27, py32, py33 deps = pytest commands = - py.test [] + py.test jedi [testenv:py25] deps = simplejson @@ -14,3 +14,8 @@ deps = deps = unittest2 {[testenv]deps} +[testenv:py33] +deps = + pytest-cov +commands = + py.test --cov jedi From af22409059e0cb420628c3f97a0f9945d37ea087 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 10 Apr 2013 17:08:20 +0200 Subject: [PATCH 091/109] Separate tox env for coverage check --- .travis.yml | 3 ++- tox.ini | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8c29ecb1..333bcc1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,9 @@ env: - TOXENV=py27 - TOXENV=py32 - TOXENV=py33 + - TOXENV=cov install: - pip install --quiet --use-mirrors tox coveralls script: - tox - - if [ $TOXENV == "py33" ]; then coveralls; fi + - if [ $TOXENV == "cov" ]; then coveralls; fi diff --git a/tox.ini b/tox.ini index c6bf7e98..205c8594 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = py25, py26, py27, py32, py33 deps = pytest commands = - py.test jedi + py.test [] [testenv:py25] deps = simplejson @@ -14,8 +14,9 @@ deps = deps = unittest2 {[testenv]deps} -[testenv:py33] +[testenv:cov] deps = pytest-cov + {[testenv]deps} commands = - py.test --cov jedi + py.test --cov jedi [] From 8e4a76ca1b41a1fd4d87140882b83d178563b69c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 10 Apr 2013 17:09:36 +0200 Subject: [PATCH 092/109] Allow TOXENV=cov to fail in travis-ci --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 333bcc1b..03af3394 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,10 @@ env: - TOXENV=py32 - TOXENV=py33 - TOXENV=cov +matrix: + allow_failures: + - env: + - TOXENV=cov install: - pip install --quiet --use-mirrors tox coveralls script: From 658bcb6421770ab0e6a622bce2da0e5962224eb9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 10 Apr 2013 17:23:50 +0200 Subject: [PATCH 093/109] Install coveralls only when needed --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 03af3394..57ea5c5e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,7 +11,11 @@ matrix: - env: - TOXENV=cov install: - - pip install --quiet --use-mirrors tox coveralls + - pip install --quiet --use-mirrors tox script: - tox - - if [ $TOXENV == "cov" ]; then coveralls; fi +after_script: + - if [ $TOXENV == "cov" ]; then + pip install --quiet --use-mirrors coveralls + coveralls; + fi From 944b3200a0ffca18d80159fc919756962f3ca990 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Wed, 10 Apr 2013 17:28:15 +0200 Subject: [PATCH 094/109] Fix allow_failures syntax allow_failures.env requires a string when matrix/global is not used? Also, indent is changed to fit with other entries. --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 57ea5c5e..51d30e2d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,9 +7,8 @@ env: - TOXENV=py33 - TOXENV=cov matrix: - allow_failures: - - env: - - TOXENV=cov + allow_failures: + - env: TOXENV=cov install: - pip install --quiet --use-mirrors tox script: From 26cabd581c06782c62ef4419bf7de966e946e780 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 10 Apr 2013 22:20:37 +0200 Subject: [PATCH 095/109] Added coverage report to README --- README.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.rst b/README.rst index b00ea40f..dcc654dd 100644 --- a/README.rst +++ b/README.rst @@ -6,6 +6,11 @@ Jedi - an awesome autocompletion library for Python :target: http://travis-ci.org/davidhalter/jedi :alt: Travis-CI build status +.. image:: https://coveralls.io/repos/davidhalter/jedi/badge.png?branch=master + :target: https://coveralls.io/r/davidhalter/jedi + :alt: Coverage Status + + Jedi is an autocompletion tool for Python that can be used in IDEs/editors. Jedi works. Jedi is fast. It understands all of the basic Python syntax elements including many builtin functions. From 1a41f92a0dde1b80b82f939982bedb15cc99b926 Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 10 Apr 2013 23:41:25 +0200 Subject: [PATCH 096/109] Added .coveragerc (refs #196) --- .coveragerc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..c48bafc2 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,18 @@ +[run] +omit = + jedi/_compatibility.py + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: From fb96f9bf3d1c8029ae032ffcf8b532329d57895d Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 10 Apr 2013 23:52:57 +0200 Subject: [PATCH 097/109] Missing semicolon in .travis.yml --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 51d30e2d..16d38fab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,6 +15,6 @@ script: - tox after_script: - if [ $TOXENV == "cov" ]; then - pip install --quiet --use-mirrors coveralls + pip install --quiet --use-mirrors coveralls; coveralls; fi From 24573c5ef2ee5f64af5a999b16ffc7db49ea993f Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Tue, 16 Apr 2013 22:51:41 +0200 Subject: [PATCH 098/109] Extended testing information in README --- README.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index dcc654dd..9ed17efe 100644 --- a/README.rst +++ b/README.rst @@ -96,10 +96,20 @@ information. Testing ======= -To run the tests:: +The test suite depends on ``tox`` and ``pytest``:: pip install tox pytest + +To run the tests for all supported Python versions:: + PIP_INSECURE=t tox +If you want to test only a specific Python version (e.g. Python 2.7), it's as +easy as :: + + tox -e py27 + +The ``PIP_INSECURE=t`` env variable is only needed for the ``py25`` target. + Tests are also run automatically on `Travis CI `_. From f98b0d7b6f36096b96d209498dfab657157c446f Mon Sep 17 00:00:00 2001 From: Danilo Bargen Date: Wed, 17 Apr 2013 23:28:37 +0200 Subject: [PATCH 099/109] New "with ignored(*exceptions)" context manager --- jedi/api_classes.py | 10 ++++------ jedi/cache.py | 9 +++------ jedi/common.py | 10 ++++++++++ jedi/dynamic.py | 5 ++--- jedi/evaluate.py | 21 ++++++--------------- jedi/evaluate_representation.py | 10 ++++------ jedi/helpers.py | 11 +++++------ jedi/imports.py | 10 +++------- jedi/keywords.py | 7 ++++--- jedi/modules.py | 13 ++++--------- jedi/parsing.py | 5 ++--- jedi/parsing_representation.py | 6 ++---- jedi/refactoring.py | 7 ++----- 13 files changed, 51 insertions(+), 73 deletions(-) diff --git a/jedi/api_classes.py b/jedi/api_classes.py index 842fe195..77d4a489 100644 --- a/jedi/api_classes.py +++ b/jedi/api_classes.py @@ -3,6 +3,7 @@ The :mod:`api_classes` module contains the return classes of the API. These classes are the much bigger part of the whole API, because they contain the interesting information about completion and goto operations. """ +from __future__ import with_statement import re import os @@ -11,6 +12,7 @@ import functools from jedi._compatibility import unicode, next from jedi import settings +from jedi import common from jedi import parsing_representation as pr from jedi import cache import keywords @@ -144,10 +146,8 @@ class BaseDefinition(object): if not isinstance(self.definition, keywords.Keyword): par = self.definition while par is not None: - try: + with common.ignored(AttributeError): path.insert(0, par.name) - except AttributeError: - pass par = par.parent return path @@ -295,10 +295,8 @@ class BaseDefinition(object): if not path: return None # for keywords the path is empty - try: + with common.ignored(KeyError): path[0] = self._mapping[path[0]] - except KeyError: - pass for key, repl in self._tuple_mapping.items(): if tuple(path[:len(key)]) == key: path = [repl] + path[len(key):] diff --git a/jedi/cache.py b/jedi/cache.py index 5d76f24e..e2c0c236 100644 --- a/jedi/cache.py +++ b/jedi/cache.py @@ -30,6 +30,7 @@ import shutil from jedi._compatibility import json from jedi import settings +from jedi import common from jedi import debug # memoize caches will be deleted after every action @@ -148,12 +149,10 @@ def cache_function_definition(stmt): def cache_star_import(func): def wrapper(scope, *args, **kwargs): - try: + with common.ignored(KeyError): mods = star_import_cache[scope] if mods[0] + settings.star_import_cache_validity > time.time(): return mods[1] - except KeyError: - pass # cache is too old and therefore invalid or not available invalidate_star_import_cache(scope) mods = func(scope, *args, **kwargs) @@ -165,15 +164,13 @@ def cache_star_import(func): def invalidate_star_import_cache(module, only_main=False): """ Important if some new modules are being reparsed """ - try: + with common.ignored(KeyError): t, mods = star_import_cache[module] del star_import_cache[module] for m in mods: invalidate_star_import_cache(m, only_main=True) - except KeyError: - pass if not only_main: # We need a list here because otherwise the list is being changed diff --git a/jedi/common.py b/jedi/common.py index 08eadd6b..ebd1f25d 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -163,3 +163,13 @@ def indent_block(text, indention=' '): text = text[:-1] lines = text.split('\n') return '\n'.join(map(lambda s: indention + s, lines)) + temp + + +@contextlib.contextmanager +def ignored(*exceptions): + """Context manager that ignores all of the specified exceptions. This will + be in the standard library starting with Python 3.4.""" + try: + yield + except exceptions: + pass diff --git a/jedi/dynamic.py b/jedi/dynamic.py index 8cc0b3eb..bd87eeb2 100644 --- a/jedi/dynamic.py +++ b/jedi/dynamic.py @@ -59,6 +59,7 @@ from jedi import cache from jedi import parsing_representation as pr from jedi import modules from jedi import settings +from jedi import common from jedi import debug from jedi import fast_parser import api_classes @@ -487,10 +488,8 @@ def related_name_add_import_modules(definitions, search_name): for d in definitions: if isinstance(d.parent, pr.Import): s = imports.ImportPath(d.parent, direct_resolve=True) - try: + with common.ignored(IndexError): new.add(s.follow(is_goto=True)[0]) - except IndexError: - pass return set(definitions) | new diff --git a/jedi/evaluate.py b/jedi/evaluate.py index c76bbb27..87d59afa 100644 --- a/jedi/evaluate.py +++ b/jedi/evaluate.py @@ -68,6 +68,7 @@ backtracking algorithm. .. todo:: nonlocal statement, needed or can be ignored? (py3k) """ +from __future__ import with_statement import sys import itertools @@ -429,11 +430,9 @@ def find_name(scope, name_str, position=None, search_global=False, if isinstance(scope, (er.Instance, er.Class)) \ and hasattr(r, 'get_descriptor_return'): # handle descriptors - try: + with common.ignored(KeyError): res_new += r.get_descriptor_return(scope) continue - except KeyError: - pass res_new.append(r) return res_new @@ -462,19 +461,15 @@ def check_getattr(inst, name_str): # str is important to lose the NamePart! module = builtin.Builtin.scope name = pr.Call(module, str(name_str), pr.Call.STRING, (0, 0), inst) - try: + with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattr__', [name]) - except KeyError: - pass if not result: # this is a little bit special. `__getattribute__` is executed # before anything else. But: I know no use case, where this # could be practical and the jedi would return wrong types. If # you ever have something, let me know! - try: + with common.ignored(KeyError): result = inst.execute_subscope_by_name('__getattribute__', [name]) - except KeyError: - pass return result @@ -536,10 +531,8 @@ def assign_tuples(tup, results, seek_name): debug.warning("invalid tuple lookup %s of result %s in %s" % (tup, results, seek_name)) else: - try: + with common.ignored(IndexError): types += func(index) - except IndexError: - pass return types result = [] @@ -648,11 +641,9 @@ def follow_call_list(call_list, follow_array=False): call = next(calls_iterator) except StopIteration: break - try: + with common.ignored(AttributeError): if str(call.name) == 'else': break - except AttributeError: - pass continue result += follow_call(call) elif call == '*': diff --git a/jedi/evaluate_representation.py b/jedi/evaluate_representation.py index 3b5c972f..d5402e4b 100644 --- a/jedi/evaluate_representation.py +++ b/jedi/evaluate_representation.py @@ -9,6 +9,8 @@ instantiated. This class represents these cases. So, why is there also a ``Class`` class here? Well, there are decorators and they change classes in Python 3. """ +from __future__ import with_statement + import copy import itertools @@ -61,10 +63,8 @@ class Instance(use_metaclass(cache.CachedMetaClass, Executable)): else: # need to execute the __init__ function, because the dynamic param # searching needs it. - try: + with common.ignored(KeyError): self.execute_subscope_by_name('__init__', self.var_args) - except KeyError: - pass # Generated instances are classes that are just generated by self # (No var_args) used. self.is_generated = False @@ -804,10 +804,8 @@ class Array(use_metaclass(cache.CachedMetaClass, pr.Base)): if isinstance(index, Instance) \ and str(index.name) in ['int', 'str'] \ and len(index.var_args) == 1: - try: + with common.ignored(KeyError, IndexError): return self.get_exact_index_types(index.var_args[0]) - except (KeyError, IndexError): - pass result = list(self._follow_values(self._array.values)) result += dynamic.check_array_additions(self) diff --git a/jedi/helpers.py b/jedi/helpers.py index c4532961..a61027c5 100644 --- a/jedi/helpers.py +++ b/jedi/helpers.py @@ -1,5 +1,8 @@ +from __future__ import with_statement + import copy +from jedi import common from jedi import parsing_representation as pr @@ -21,13 +24,11 @@ def fast_parent_copy(obj): before = () for cls in new_obj.__class__.__mro__: - try: + with common.ignored(AttributeError): if before == cls.__slots__: continue before = cls.__slots__ items += [(n, getattr(new_obj, n)) for n in before] - except AttributeError: - pass for key, value in items: # replace parent (first try _parent and then parent) @@ -35,10 +36,8 @@ def fast_parent_copy(obj): if key == 'parent' and '_parent' in items: # parent can be a property continue - try: + with common.ignored(KeyError): setattr(new_obj, key, new_elements[value]) - except KeyError: - pass elif key in ['parent_function', 'use_as_parent', '_sub_module']: continue elif isinstance(value, list): diff --git a/jedi/imports.py b/jedi/imports.py index db72925f..d6081386 100644 --- a/jedi/imports.py +++ b/jedi/imports.py @@ -11,7 +11,6 @@ correct implementation is delegated to _compatibility. This module also supports import autocompletion, which means to complete statements like ``from datetim`` (curser at the end would return ``datetime``). """ - from __future__ import with_statement import os @@ -21,6 +20,7 @@ import itertools from jedi._compatibility import find_module from jedi import modules +from jedi import common from jedi import debug from jedi import parsing_representation as pr from jedi import cache @@ -122,11 +122,9 @@ class ImportPath(pr.Base): if self.import_stmt.relative_count: rel_path = self.get_relative_path() + '/__init__.py' - try: + with common.ignored(IOError): m = modules.Module(rel_path) names += m.parser.module.get_defined_names() - except IOError: - pass else: if on_import_stmt and isinstance(scope, pr.Module) \ and scope.path.endswith('__init__.py'): @@ -274,10 +272,8 @@ class ImportPath(pr.Base): and len(self.import_path) == 1: # follow `from . import some_variable` rel_path = self.get_relative_path() - try: + with common.ignored(ImportError): current_namespace = follow_str(rel_path, '__init__') - except ImportError: - pass if current_namespace[1]: rest = self.import_path[i:] else: diff --git a/jedi/keywords.py b/jedi/keywords.py index 5495ae32..cd591aa6 100644 --- a/jedi/keywords.py +++ b/jedi/keywords.py @@ -1,7 +1,10 @@ +from __future__ import with_statement + import pydoc import keyword from jedi._compatibility import is_py3k +from jedi import common import builtin try: @@ -63,12 +66,10 @@ def imitate_pydoc(string): # with unicode strings) string = str(string) h = pydoc.help - try: + with common.ignored(KeyError): # try to access symbols string = h.symbols[string] string, _, related = string.partition(' ') - except KeyError: - pass get_target = lambda s: h.topics.get(s, h.keywords.get(s)) while isinstance(string, str): diff --git a/jedi/modules.py b/jedi/modules.py index 9277e742..e67cc2c3 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -27,6 +27,7 @@ from jedi import parsing_representation as pr from jedi import fast_parser from jedi import debug from jedi import settings +from jedi import common class CachedModule(object): @@ -107,11 +108,9 @@ class ModuleWithCursor(Module): def parser(self): """ get the parser lazy """ if not self._parser: - try: + with common.ignored(KeyError): parser = cache.parser_cache[self.path].parser cache.invalidate_star_import_cache(parser.module) - except KeyError: - pass # Call the parser already here, because it will be used anyways. # Also, the position is here important (which will not be used by # default), therefore fill the cache here. @@ -348,10 +347,8 @@ def sys_path_with_modifications(module): return [] # support for modules without a path is intentionally bad. curdir = os.path.abspath(os.curdir) - try: + with common.ignored(OSError): os.chdir(os.path.dirname(module.path)) - except OSError: - pass result = check_module(module) result += detect_django_path(module.path) @@ -372,12 +369,10 @@ def detect_django_path(module_path): else: module_path = new - try: + with common.ignored(IOError): with open(module_path + os.path.sep + 'manage.py'): debug.dbg('Found django path: %s' % module_path) result.append(module_path) - except IOError: - pass return result diff --git a/jedi/parsing.py b/jedi/parsing.py index 69607fec..e5279f71 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -15,6 +15,7 @@ within the statement. This lowers memory usage and cpu time and reduces the complexity of the ``Parser`` (there's another parser sitting inside ``Statement``, which produces ``Array`` and ``Call``). """ +from __future__ import with_statement import tokenize import keyword @@ -396,7 +397,7 @@ class Parser(object): self._check_user_stmt(stmt) # Attribute docstring (PEP 257) support - try: + with common.ignored(IndexError, AttributeError): # If string literal is being parsed first_tok = stmt.token_list[0] if (not stmt.set_vars and @@ -405,8 +406,6 @@ class Parser(object): first_tok[0] == tokenize.STRING): # ... then set it as a docstring self.scope.statements[-1].add_docstr(first_tok[1]) - except (IndexError, AttributeError): - pass if tok in always_break + not_first_break: self._gen.push_last_back() diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index 311f38ab..d4b8380c 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -34,6 +34,7 @@ statements in this scope. Check this out: See also :attr:`Scope.subscopes` and :attr:`Scope.statements`. """ +from __future__ import with_statement import os import re @@ -1280,11 +1281,8 @@ class Array(Call): inner = [] for i, stmt in enumerate(self.values): s = '' - try: + with common.ignored(IndexError): key = self.keys[i] - except IndexError: - pass - else: s += key.get_code(new_line=False) + ': ' s += stmt.get_code(new_line=False) inner.append(s) diff --git a/jedi/refactoring.py b/jedi/refactoring.py index a724ec16..ced1a732 100644 --- a/jedi/refactoring.py +++ b/jedi/refactoring.py @@ -12,11 +12,11 @@ following functions (sometimes bug-prone): - extract variable - inline variable """ - from __future__ import with_statement import difflib +from jedi import common from jedi import modules from jedi import helpers from jedi import parsing_representation as pr @@ -168,7 +168,7 @@ def inline(script): dct = {} definitions = script.goto() - try: + with common.ignored(AssertionError): assert len(definitions) == 1 stmt = definitions[0].definition related_names = script.related_names() @@ -202,7 +202,4 @@ def inline(script): else: new_lines.pop(index) - except AssertionError: - pass - return Refactoring(dct) From a28bc7195f4f690de4e60be27bf6c364df4642fb Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 19 Apr 2013 23:50:28 +0430 Subject: [PATCH 100/109] added tokenizer of the standard library to jedi, because in the future it shall replace tokenize --- jedi/tokenizer.py | 556 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 556 insertions(+) create mode 100644 jedi/tokenizer.py diff --git a/jedi/tokenizer.py b/jedi/tokenizer.py new file mode 100644 index 00000000..f575e9bc --- /dev/null +++ b/jedi/tokenizer.py @@ -0,0 +1,556 @@ +"""Tokenization help for Python programs. + +tokenize(readline) is a generator that breaks a stream of bytes into +Python tokens. It decodes the bytes according to PEP-0263 for +determining source file encoding. + +It accepts a readline-like method which is called repeatedly to get the +next line of input (or b"" for EOF). It generates 5-tuples with these +members: + + the token type (see token.py) + the token (a string) + the starting (row, column) indices of the token (a 2-tuple of ints) + the ending (row, column) indices of the token (a 2-tuple of ints) + the original line (string) + +It is designed to match the working of the Python tokenizer exactly, except +that it produces COMMENT tokens for comments and gives type OP for all +operators. Additionally, all token lists start with an ENCODING token +which tells you which encoding was used to decode the bytes stream. +""" + +__author__ = 'Ka-Ping Yee ' +__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' + 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' + 'Michael Foord') +import builtins +import re +import sys +from token import * +from codecs import lookup, BOM_UTF8 +import collections +from io import TextIOWrapper +cookie_re = re.compile("coding[:=]\s*([-\w.]+)") + +import token +__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding", + "NL", "untokenize", "ENCODING", "TokenInfo"] +del token + +COMMENT = N_TOKENS +tok_name[COMMENT] = 'COMMENT' +NL = N_TOKENS + 1 +tok_name[NL] = 'NL' +ENCODING = N_TOKENS + 2 +tok_name[ENCODING] = 'ENCODING' +N_TOKENS += 3 + +class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): + def __repr__(self): + annotated_type = '%d (%s)' % (self.type, tok_name[self.type]) + return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % + self._replace(type=annotated_type)) + +def group(*choices): return '(' + '|'.join(choices) + ')' +def any(*choices): return group(*choices) + '*' +def maybe(*choices): return group(*choices) + '?' + +# Note: we use unicode matching for names ("\w") but ascii matching for +# number literals. +Whitespace = r'[ \f\t]*' +Comment = r'#[^\r\n]*' +Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) +Name = r'\w+' + +Hexnumber = r'0[xX][0-9a-fA-F]+' +Binnumber = r'0[bB][01]+' +Octnumber = r'0[oO][0-7]+' +Decnumber = r'(?:0+|[1-9][0-9]*)' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?[0-9]+' +Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent) +Expfloat = r'[0-9]+' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]') +Number = group(Imagnumber, Floatnumber, Intnumber) + +# Tail end of ' string. +Single = r"[^'\\]*(?:\\.[^'\\]*)*'" +# Tail end of " string. +Double = r'[^"\\]*(?:\\.[^"\\]*)*"' +# Tail end of ''' string. +Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" +# Tail end of """ string. +Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' +Triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""') +# Single-line ' or " string. +String = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') + +# Because of leftmost-then-longest match semantics, be sure to put the +# longest operators first (e.g., if = came before ==, == would get +# recognized as two instances of =). +Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=", + r"//=?", r"->", + r"[+\-*/%&|^=<>]=?", + r"~") + +Bracket = '[][(){}]' +Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]') +Funny = group(Operator, Bracket, Special) + +PlainToken = group(Number, Funny, String, Name) +Token = Ignore + PlainToken + +# First (or only) line of ' or " string. +ContStr = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + + group("'", r'\\\r?\n'), + r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + group('"', r'\\\r?\n')) +PseudoExtras = group(r'\\\r?\n', Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +def _compile(expr): + return re.compile(expr, re.UNICODE) + +tokenprog, pseudoprog, single3prog, double3prog = map( + _compile, (Token, PseudoToken, Single3, Double3)) +endprogs = {"'": _compile(Single), '"': _compile(Double), + "'''": single3prog, '"""': double3prog, + "r'''": single3prog, 'r"""': double3prog, + "b'''": single3prog, 'b"""': double3prog, + "br'''": single3prog, 'br"""': double3prog, + "R'''": single3prog, 'R"""': double3prog, + "B'''": single3prog, 'B"""': double3prog, + "bR'''": single3prog, 'bR"""': double3prog, + "Br'''": single3prog, 'Br"""': double3prog, + "BR'''": single3prog, 'BR"""': double3prog, + 'r': None, 'R': None, 'b': None, 'B': None} + +triple_quoted = {} +for t in ("'''", '"""', + "r'''", 'r"""', "R'''", 'R"""', + "b'''", 'b"""', "B'''", 'B"""', + "br'''", 'br"""', "Br'''", 'Br"""', + "bR'''", 'bR"""', "BR'''", 'BR"""'): + triple_quoted[t] = t +single_quoted = {} +for t in ("'", '"', + "r'", 'r"', "R'", 'R"', + "b'", 'b"', "B'", 'B"', + "br'", 'br"', "Br'", 'Br"', + "bR'", 'bR"', "BR'", 'BR"' ): + single_quoted[t] = t + +del _compile + +tabsize = 8 + +class TokenError(Exception): pass + +class StopTokenizing(Exception): pass + + +class Untokenizer: + + def __init__(self): + self.tokens = [] + self.prev_row = 1 + self.prev_col = 0 + self.encoding = None + + def add_whitespace(self, start): + row, col = start + assert row <= self.prev_row + col_offset = col - self.prev_col + if col_offset: + self.tokens.append(" " * col_offset) + + def untokenize(self, iterable): + for t in iterable: + if len(t) == 2: + self.compat(t, iterable) + break + tok_type, token, start, end, line = t + if tok_type == ENCODING: + self.encoding = token + continue + self.add_whitespace(start) + self.tokens.append(token) + self.prev_row, self.prev_col = end + if tok_type in (NEWLINE, NL): + self.prev_row += 1 + self.prev_col = 0 + return "".join(self.tokens) + + def compat(self, token, iterable): + startline = False + indents = [] + toks_append = self.tokens.append + toknum, tokval = token + + if toknum in (NAME, NUMBER): + tokval += ' ' + if toknum in (NEWLINE, NL): + startline = True + prevstring = False + for tok in iterable: + toknum, tokval = tok[:2] + if toknum == ENCODING: + self.encoding = tokval + continue + + if toknum in (NAME, NUMBER): + tokval += ' ' + + # Insert a space between two consecutive strings + if toknum == STRING: + if prevstring: + tokval = ' ' + tokval + prevstring = True + else: + prevstring = False + + if toknum == INDENT: + indents.append(tokval) + continue + elif toknum == DEDENT: + indents.pop() + continue + elif toknum in (NEWLINE, NL): + startline = True + elif startline and indents: + toks_append(indents[-1]) + startline = False + toks_append(tokval) + + +def untokenize(iterable): + """Transform tokens back into Python source code. + It returns a bytes object, encoded using the ENCODING + token, which is the first token sequence output by tokenize. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + Round-trip invariant for full input: + Untokenized source will match input source exactly + + Round-trip invariant for limited intput: + # Output bytes will tokenize the back to the input + t1 = [tok[:2] for tok in tokenize(f.readline)] + newcode = untokenize(t1) + readline = BytesIO(newcode).readline + t2 = [tok[:2] for tok in tokenize(readline)] + assert t1 == t2 + """ + ut = Untokenizer() + out = ut.untokenize(iterable) + if ut.encoding is not None: + out = out.encode(ut.encoding) + return out + + +def _get_normal_name(orig_enc): + """Imitates get_normal_name in tokenizer.c.""" + # Only care about the first 12 characters. + enc = orig_enc[:12].lower().replace("_", "-") + if enc == "utf-8" or enc.startswith("utf-8-"): + return "utf-8" + if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ + enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): + return "iso-8859-1" + return orig_enc + +def detect_encoding(readline): + """ + The detect_encoding() function is used to detect the encoding that should + be used to decode a Python source file. It requires one argment, readline, + in the same way as the tokenize() generator. + + It will call readline a maximum of twice, and return the encoding used + (as a string) and a list of any lines (left as bytes) it has read in. + + It detects the encoding from the presence of a utf-8 bom or an encoding + cookie as specified in pep-0263. If both a bom and a cookie are present, + but disagree, a SyntaxError will be raised. If the encoding cookie is an + invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, + 'utf-8-sig' is returned. + + If no encoding is specified, then the default of 'utf-8' will be returned. + """ + bom_found = False + encoding = None + default = 'utf-8' + def read_or_stop(): + try: + return readline() + except StopIteration: + return b'' + + def find_cookie(line): + try: + line_string = line.decode('ascii') + except UnicodeDecodeError: + return None + + matches = cookie_re.findall(line_string) + if not matches: + return None + encoding = _get_normal_name(matches[0]) + try: + codec = lookup(encoding) + except LookupError: + # This behaviour mimics the Python interpreter + raise SyntaxError("unknown encoding: " + encoding) + + if bom_found: + if codec.name != 'utf-8': + # This behaviour mimics the Python interpreter + raise SyntaxError('encoding problem: utf-8') + encoding += '-sig' + return encoding + + first = read_or_stop() + if first.startswith(BOM_UTF8): + bom_found = True + first = first[3:] + default = 'utf-8-sig' + if not first: + return default, [] + + encoding = find_cookie(first) + if encoding: + return encoding, [first] + + second = read_or_stop() + if not second: + return default, [first] + + encoding = find_cookie(second) + if encoding: + return encoding, [first, second] + + return default, [first, second] + + +def open(filename): + """Open a file in read only mode using the encoding detected by + detect_encoding(). + """ + buffer = builtins.open(filename, 'rb') + encoding, lines = detect_encoding(buffer.readline) + buffer.seek(0) + text = TextIOWrapper(buffer, encoding, line_buffering=True) + text.mode = 'r' + return text + + +def tokenize(readline): + """ + The tokenize() generator requires one argment, readline, which + must be a callable object which provides the same interface as the + readline() method of built-in file objects. Each call to the function + should return one line of input as bytes. Alternately, readline + can be a callable function terminating with StopIteration: + readline = open(myfile, 'rb').__next__ # Example of alternate readline + + The generator produces 5-tuples with these members: the token type; the + token string; a 2-tuple (srow, scol) of ints specifying the row and + column where the token begins in the source; a 2-tuple (erow, ecol) of + ints specifying the row and column where the token ends in the source; + and the line on which the token was found. The line passed is the + logical line; continuation lines are included. + + The first token sequence will always be an ENCODING token + which tells you which encoding was used to decode the bytes stream. + """ + # This import is here to avoid problems when the itertools module is not + # built yet and tokenize is imported. + from itertools import chain, repeat + encoding, consumed = detect_encoding(readline) + rl_gen = iter(readline, b"") + empty = repeat(b"") + return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding) + + +def _tokenize(readline, encoding): + lnum = parenlev = continued = 0 + numchars = '0123456789' + contstr, needcont = '', 0 + contline = None + indents = [0] + + if encoding is not None: + if encoding == "utf-8-sig": + # BOM will already have been stripped. + encoding = "utf-8" + yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') + while True: # loop over lines in stream + try: + line = readline() + except StopIteration: + line = b'' + + if encoding is not None: + line = line.decode(encoding) + lnum += 1 + pos, max = 0, len(line) + + if contstr: # continued string + if not line: + raise TokenError("EOF in multi-line string", strstart) + endmatch = endprog.match(line) + if endmatch: + pos = end = endmatch.end(0) + yield TokenInfo(STRING, contstr + line[:end], + strstart, (lnum, end), contline + line) + contstr, needcont = '', 0 + contline = None + elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': + yield TokenInfo(ERRORTOKEN, contstr + line, + strstart, (lnum, len(line)), contline) + contstr = '' + contline = None + continue + else: + contstr = contstr + line + contline = contline + line + continue + + elif parenlev == 0 and not continued: # new statement + if not line: break + column = 0 + while pos < max: # measure leading whitespace + if line[pos] == ' ': + column += 1 + elif line[pos] == '\t': + column = (column//tabsize + 1)*tabsize + elif line[pos] == '\f': + column = 0 + else: + break + pos += 1 + if pos == max: + break + + if line[pos] in '#\r\n': # skip comments or blank lines + if line[pos] == '#': + comment_token = line[pos:].rstrip('\r\n') + nl_pos = pos + len(comment_token) + yield TokenInfo(COMMENT, comment_token, + (lnum, pos), (lnum, pos + len(comment_token)), line) + yield TokenInfo(NL, line[nl_pos:], + (lnum, nl_pos), (lnum, len(line)), line) + else: + yield TokenInfo((NL, COMMENT)[line[pos] == '#'], line[pos:], + (lnum, pos), (lnum, len(line)), line) + continue + + if column > indents[-1]: # count indents or dedents + indents.append(column) + yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + while column < indents[-1]: + if column not in indents: + raise IndentationError( + "unindent does not match any outer indentation level", + ("", lnum, pos, line)) + indents = indents[:-1] + yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) + + else: # continued statement + if not line: + raise TokenError("EOF in multi-line statement", (lnum, 0)) + continued = 0 + + while pos < max: + pseudomatch = pseudoprog.match(line, pos) + if pseudomatch: # scan for tokens + start, end = pseudomatch.span(1) + spos, epos, pos = (lnum, start), (lnum, end), end + token, initial = line[start:end], line[start] + + if (initial in numchars or # ordinary number + (initial == '.' and token != '.' and token != '...')): + yield TokenInfo(NUMBER, token, spos, epos, line) + elif initial in '\r\n': + yield TokenInfo(NL if parenlev > 0 else NEWLINE, + token, spos, epos, line) + elif initial == '#': + assert not token.endswith("\n") + yield TokenInfo(COMMENT, token, spos, epos, line) + elif token in triple_quoted: + endprog = endprogs[token] + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + pos = endmatch.end(0) + token = line[start:pos] + yield TokenInfo(STRING, token, spos, (lnum, pos), line) + else: + strstart = (lnum, start) # multiple lines + contstr = line[start:] + contline = line + break + elif initial in single_quoted or \ + token[:2] in single_quoted or \ + token[:3] in single_quoted: + if token[-1] == '\n': # continued string + strstart = (lnum, start) + endprog = (endprogs[initial] or endprogs[token[1]] or + endprogs[token[2]]) + contstr, needcont = line[start:], 1 + contline = line + break + else: # ordinary string + yield TokenInfo(STRING, token, spos, epos, line) + elif initial.isidentifier(): # ordinary name + yield TokenInfo(NAME, token, spos, epos, line) + elif initial == '\\': # continued stmt + continued = 1 + else: + if initial in '([{': + parenlev += 1 + elif initial in ')]}': + parenlev -= 1 + yield TokenInfo(OP, token, spos, epos, line) + else: + yield TokenInfo(ERRORTOKEN, line[pos], + (lnum, pos), (lnum, pos+1), line) + pos += 1 + + for indent in indents[1:]: # pop remaining indent levels + yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') + yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') + + +# An undocumented, backwards compatible, API for all the places in the standard +# library that expect to be able to use tokenize with strings +def generate_tokens(readline): + return _tokenize(readline, None) + +if __name__ == "__main__": + # Quick sanity check + s = b'''def parseline(self, line): + """Parse the line into a command name and a string containing + the arguments. Returns a tuple containing (command, args, line). + 'command' and 'args' may be None if the line couldn't be parsed. + """ + line = line.strip() + if not line: + return None, None, line + elif line[0] == '?': + line = 'help ' + line[1:] + elif line[0] == '!': + if hasattr(self, 'do_shell'): + line = 'shell ' + line[1:] + else: + return None, None, line + i, n = 0, len(line) + while i < n and line[i] in self.identchars: i = i+1 + cmd, arg = line[:i], line[i:].strip() + return cmd, arg, line + ''' + for tok in tokenize(iter(s.splitlines()).__next__): + print(tok) From 3b0f4b87cfe40e112177ea8a39a18ef102cac828 Mon Sep 17 00:00:00 2001 From: David Halter Date: Fri, 19 Apr 2013 23:59:33 +0430 Subject: [PATCH 101/109] removed unnecessary stuff for the jedi tokenizer --- jedi/tokenizer.py | 164 +++------------------------------------------- 1 file changed, 8 insertions(+), 156 deletions(-) diff --git a/jedi/tokenizer.py b/jedi/tokenizer.py index f575e9bc..90972c85 100644 --- a/jedi/tokenizer.py +++ b/jedi/tokenizer.py @@ -1,4 +1,11 @@ -"""Tokenization help for Python programs. +""" +This tokenizer has been copied from the ``tokenize.py`` standard library +tokenizer. The reason was simple: The standanrd library tokenizer fails +if the indentation is not right. The fast parser of jedi however requires +"wrong" indentation. + +Tokenization help for Python programs. +++++++++++++++++++++++++++++++++++++++ tokenize(readline) is a generator that breaks a stream of bytes into Python tokens. It decodes the bytes according to PEP-0263 for @@ -20,23 +27,12 @@ operators. Additionally, all token lists start with an ENCODING token which tells you which encoding was used to decode the bytes stream. """ -__author__ = 'Ka-Ping Yee ' -__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' - 'Skip Montanaro, Raymond Hettinger, Trent Nelson, ' - 'Michael Foord') -import builtins import re -import sys from token import * from codecs import lookup, BOM_UTF8 import collections -from io import TextIOWrapper cookie_re = re.compile("coding[:=]\s*([-\w.]+)") -import token -__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding", - "NL", "untokenize", "ENCODING", "TokenInfo"] -del token COMMENT = N_TOKENS tok_name[COMMENT] = 'COMMENT' @@ -152,107 +148,6 @@ class TokenError(Exception): pass class StopTokenizing(Exception): pass -class Untokenizer: - - def __init__(self): - self.tokens = [] - self.prev_row = 1 - self.prev_col = 0 - self.encoding = None - - def add_whitespace(self, start): - row, col = start - assert row <= self.prev_row - col_offset = col - self.prev_col - if col_offset: - self.tokens.append(" " * col_offset) - - def untokenize(self, iterable): - for t in iterable: - if len(t) == 2: - self.compat(t, iterable) - break - tok_type, token, start, end, line = t - if tok_type == ENCODING: - self.encoding = token - continue - self.add_whitespace(start) - self.tokens.append(token) - self.prev_row, self.prev_col = end - if tok_type in (NEWLINE, NL): - self.prev_row += 1 - self.prev_col = 0 - return "".join(self.tokens) - - def compat(self, token, iterable): - startline = False - indents = [] - toks_append = self.tokens.append - toknum, tokval = token - - if toknum in (NAME, NUMBER): - tokval += ' ' - if toknum in (NEWLINE, NL): - startline = True - prevstring = False - for tok in iterable: - toknum, tokval = tok[:2] - if toknum == ENCODING: - self.encoding = tokval - continue - - if toknum in (NAME, NUMBER): - tokval += ' ' - - # Insert a space between two consecutive strings - if toknum == STRING: - if prevstring: - tokval = ' ' + tokval - prevstring = True - else: - prevstring = False - - if toknum == INDENT: - indents.append(tokval) - continue - elif toknum == DEDENT: - indents.pop() - continue - elif toknum in (NEWLINE, NL): - startline = True - elif startline and indents: - toks_append(indents[-1]) - startline = False - toks_append(tokval) - - -def untokenize(iterable): - """Transform tokens back into Python source code. - It returns a bytes object, encoded using the ENCODING - token, which is the first token sequence output by tokenize. - - Each element returned by the iterable must be a token sequence - with at least two elements, a token number and token value. If - only two tokens are passed, the resulting output is poor. - - Round-trip invariant for full input: - Untokenized source will match input source exactly - - Round-trip invariant for limited intput: - # Output bytes will tokenize the back to the input - t1 = [tok[:2] for tok in tokenize(f.readline)] - newcode = untokenize(t1) - readline = BytesIO(newcode).readline - t2 = [tok[:2] for tok in tokenize(readline)] - assert t1 == t2 - """ - ut = Untokenizer() - out = ut.untokenize(iterable) - if ut.encoding is not None: - out = out.encode(ut.encoding) - return out - - def _get_normal_name(orig_enc): """Imitates get_normal_name in tokenizer.c.""" # Only care about the first 12 characters. @@ -336,18 +231,6 @@ def detect_encoding(readline): return default, [first, second] -def open(filename): - """Open a file in read only mode using the encoding detected by - detect_encoding(). - """ - buffer = builtins.open(filename, 'rb') - encoding, lines = detect_encoding(buffer.readline) - buffer.seek(0) - text = TextIOWrapper(buffer, encoding, line_buffering=True) - text.mode = 'r' - return text - - def tokenize(readline): """ The tokenize() generator requires one argment, readline, which @@ -523,34 +406,3 @@ def _tokenize(readline, encoding): for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') - - -# An undocumented, backwards compatible, API for all the places in the standard -# library that expect to be able to use tokenize with strings -def generate_tokens(readline): - return _tokenize(readline, None) - -if __name__ == "__main__": - # Quick sanity check - s = b'''def parseline(self, line): - """Parse the line into a command name and a string containing - the arguments. Returns a tuple containing (command, args, line). - 'command' and 'args' may be None if the line couldn't be parsed. - """ - line = line.strip() - if not line: - return None, None, line - elif line[0] == '?': - line = 'help ' + line[1:] - elif line[0] == '!': - if hasattr(self, 'do_shell'): - line = 'shell ' + line[1:] - else: - return None, None, line - i, n = 0, len(line) - while i < n and line[i] in self.identchars: i = i+1 - cmd, arg = line[:i], line[i:].strip() - return cmd, arg, line - ''' - for tok in tokenize(iter(s.splitlines()).__next__): - print(tok) From 06da7ddeecb068384e4e21f0e4e095e42724cf51 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 09:21:22 +0430 Subject: [PATCH 102/109] tokenizer py27 support --- jedi/tokenizer.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/jedi/tokenizer.py b/jedi/tokenizer.py index 90972c85..989678a6 100644 --- a/jedi/tokenizer.py +++ b/jedi/tokenizer.py @@ -27,12 +27,15 @@ operators. Additionally, all token lists start with an ENCODING token which tells you which encoding was used to decode the bytes stream. """ +import string import re from token import * from codecs import lookup, BOM_UTF8 import collections cookie_re = re.compile("coding[:=]\s*([-\w.]+)") +namechars = string.ascii_letters + '_' + COMMENT = N_TOKENS tok_name[COMMENT] = 'COMMENT' @@ -388,7 +391,7 @@ def _tokenize(readline, encoding): break else: # ordinary string yield TokenInfo(STRING, token, spos, epos, line) - elif initial.isidentifier(): # ordinary name + elif initial in namechars: # ordinary name yield TokenInfo(NAME, token, spos, epos, line) elif initial == '\\': # continued stmt continued = 1 @@ -400,9 +403,15 @@ def _tokenize(readline, encoding): yield TokenInfo(OP, token, spos, epos, line) else: yield TokenInfo(ERRORTOKEN, line[pos], - (lnum, pos), (lnum, pos+1), line) + (lnum, pos), (lnum, pos + 1), line) pos += 1 for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') + + +# An undocumented, backwards compatible, API for all the places in the standard +# library that expect to be able to use tokenize with strings +def generate_tokens(readline): + return _tokenize(readline, None) From 6b2619a844c56cd56c927d5be503ec2efd04c6b7 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 09:23:27 +0430 Subject: [PATCH 103/109] tokenizer removed unused functions --- jedi/tokenizer.py | 113 ---------------------------------------------- 1 file changed, 113 deletions(-) diff --git a/jedi/tokenizer.py b/jedi/tokenizer.py index 989678a6..c5ff5e2f 100644 --- a/jedi/tokenizer.py +++ b/jedi/tokenizer.py @@ -148,119 +148,6 @@ tabsize = 8 class TokenError(Exception): pass -class StopTokenizing(Exception): pass - - -def _get_normal_name(orig_enc): - """Imitates get_normal_name in tokenizer.c.""" - # Only care about the first 12 characters. - enc = orig_enc[:12].lower().replace("_", "-") - if enc == "utf-8" or enc.startswith("utf-8-"): - return "utf-8" - if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \ - enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")): - return "iso-8859-1" - return orig_enc - -def detect_encoding(readline): - """ - The detect_encoding() function is used to detect the encoding that should - be used to decode a Python source file. It requires one argment, readline, - in the same way as the tokenize() generator. - - It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (left as bytes) it has read in. - - It detects the encoding from the presence of a utf-8 bom or an encoding - cookie as specified in pep-0263. If both a bom and a cookie are present, - but disagree, a SyntaxError will be raised. If the encoding cookie is an - invalid charset, raise a SyntaxError. Note that if a utf-8 bom is found, - 'utf-8-sig' is returned. - - If no encoding is specified, then the default of 'utf-8' will be returned. - """ - bom_found = False - encoding = None - default = 'utf-8' - def read_or_stop(): - try: - return readline() - except StopIteration: - return b'' - - def find_cookie(line): - try: - line_string = line.decode('ascii') - except UnicodeDecodeError: - return None - - matches = cookie_re.findall(line_string) - if not matches: - return None - encoding = _get_normal_name(matches[0]) - try: - codec = lookup(encoding) - except LookupError: - # This behaviour mimics the Python interpreter - raise SyntaxError("unknown encoding: " + encoding) - - if bom_found: - if codec.name != 'utf-8': - # This behaviour mimics the Python interpreter - raise SyntaxError('encoding problem: utf-8') - encoding += '-sig' - return encoding - - first = read_or_stop() - if first.startswith(BOM_UTF8): - bom_found = True - first = first[3:] - default = 'utf-8-sig' - if not first: - return default, [] - - encoding = find_cookie(first) - if encoding: - return encoding, [first] - - second = read_or_stop() - if not second: - return default, [first] - - encoding = find_cookie(second) - if encoding: - return encoding, [first, second] - - return default, [first, second] - - -def tokenize(readline): - """ - The tokenize() generator requires one argment, readline, which - must be a callable object which provides the same interface as the - readline() method of built-in file objects. Each call to the function - should return one line of input as bytes. Alternately, readline - can be a callable function terminating with StopIteration: - readline = open(myfile, 'rb').__next__ # Example of alternate readline - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple (srow, scol) of ints specifying the row and - column where the token begins in the source; a 2-tuple (erow, ecol) of - ints specifying the row and column where the token ends in the source; - and the line on which the token was found. The line passed is the - logical line; continuation lines are included. - - The first token sequence will always be an ENCODING token - which tells you which encoding was used to decode the bytes stream. - """ - # This import is here to avoid problems when the itertools module is not - # built yet and tokenize is imported. - from itertools import chain, repeat - encoding, consumed = detect_encoding(readline) - rl_gen = iter(readline, b"") - empty = repeat(b"") - return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding) - def _tokenize(readline, encoding): lnum = parenlev = continued = 0 From 0d385563a5d1bb1db3e20e8fb6bec6ce72099d19 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 09:28:32 +0430 Subject: [PATCH 104/109] tokenizer: removed encoding --- jedi/tokenizer.py | 38 +++----------------------------------- 1 file changed, 3 insertions(+), 35 deletions(-) diff --git a/jedi/tokenizer.py b/jedi/tokenizer.py index c5ff5e2f..e48cf30b 100644 --- a/jedi/tokenizer.py +++ b/jedi/tokenizer.py @@ -4,27 +4,8 @@ tokenizer. The reason was simple: The standanrd library tokenizer fails if the indentation is not right. The fast parser of jedi however requires "wrong" indentation. -Tokenization help for Python programs. -++++++++++++++++++++++++++++++++++++++ - -tokenize(readline) is a generator that breaks a stream of bytes into -Python tokens. It decodes the bytes according to PEP-0263 for -determining source file encoding. - -It accepts a readline-like method which is called repeatedly to get the -next line of input (or b"" for EOF). It generates 5-tuples with these -members: - - the token type (see token.py) - the token (a string) - the starting (row, column) indices of the token (a 2-tuple of ints) - the ending (row, column) indices of the token (a 2-tuple of ints) - the original line (string) - -It is designed to match the working of the Python tokenizer exactly, except -that it produces COMMENT tokens for comments and gives type OP for all -operators. Additionally, all token lists start with an ENCODING token -which tells you which encoding was used to decode the bytes stream. +Basically this is a stripped down version of the standard library module, so +you can read the documentation there. """ import string @@ -149,26 +130,19 @@ tabsize = 8 class TokenError(Exception): pass -def _tokenize(readline, encoding): +def generate_tokens(readline): lnum = parenlev = continued = 0 numchars = '0123456789' contstr, needcont = '', 0 contline = None indents = [0] - if encoding is not None: - if encoding == "utf-8-sig": - # BOM will already have been stripped. - encoding = "utf-8" - yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '') while True: # loop over lines in stream try: line = readline() except StopIteration: line = b'' - if encoding is not None: - line = line.decode(encoding) lnum += 1 pos, max = 0, len(line) @@ -296,9 +270,3 @@ def _tokenize(readline, encoding): for indent in indents[1:]: # pop remaining indent levels yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '') yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '') - - -# An undocumented, backwards compatible, API for all the places in the standard -# library that expect to be able to use tokenize with strings -def generate_tokens(readline): - return _tokenize(readline, None) From e2a6562384b356d59690f0e1be7d765b284a4a78 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 09:51:52 +0430 Subject: [PATCH 105/109] tokenizer: removed raised errors --- jedi/tokenizer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/jedi/tokenizer.py b/jedi/tokenizer.py index e48cf30b..e4b5c0d6 100644 --- a/jedi/tokenizer.py +++ b/jedi/tokenizer.py @@ -148,7 +148,8 @@ def generate_tokens(readline): if contstr: # continued string if not line: - raise TokenError("EOF in multi-line string", strstart) + # multiline string has not been finished + break endmatch = endprog.match(line) if endmatch: pos = end = endmatch.end(0) @@ -200,16 +201,13 @@ def generate_tokens(readline): indents.append(column) yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line) while column < indents[-1]: - if column not in indents: - raise IndentationError( - "unindent does not match any outer indentation level", - ("", lnum, pos, line)) indents = indents[:-1] yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line) else: # continued statement if not line: - raise TokenError("EOF in multi-line statement", (lnum, 0)) + # basically a statement has not been finished here. + break continued = 0 while pos < max: From d4428c497ac4f3c4500d5db6f19051c099200681 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 09:52:54 +0430 Subject: [PATCH 106/109] use tokenizer instead of tokenize in all modules --- jedi/common.py | 2 +- jedi/modules.py | 2 +- jedi/parsing.py | 2 +- jedi/parsing_representation.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/jedi/common.py b/jedi/common.py index ebd1f25d..04d17a5d 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -2,7 +2,7 @@ import sys import contextlib import functools -import tokenize +import tokenizer as tokenize from jedi._compatibility import next, reraise from jedi import debug diff --git a/jedi/modules.py b/jedi/modules.py index e67cc2c3..3301eba7 100644 --- a/jedi/modules.py +++ b/jedi/modules.py @@ -16,7 +16,7 @@ Apart from those classes there's a ``sys.path`` fetching function, as well as from __future__ import with_statement import re -import tokenize +import tokenizer as tokenize import sys import os diff --git a/jedi/parsing.py b/jedi/parsing.py index e5279f71..2241b12d 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -17,7 +17,7 @@ complexity of the ``Parser`` (there's another parser sitting inside """ from __future__ import with_statement -import tokenize +import tokenizer as tokenize import keyword from jedi._compatibility import next, StringIO diff --git a/jedi/parsing_representation.py b/jedi/parsing_representation.py index d4b8380c..a7dc9545 100644 --- a/jedi/parsing_representation.py +++ b/jedi/parsing_representation.py @@ -38,7 +38,7 @@ from __future__ import with_statement import os import re -import tokenize +import tokenizer as tokenize from jedi._compatibility import next, literal_eval, cleandoc, Python3Method, \ encoding, property, unicode, is_py3k From a7308fa51d2a1442fb9011a50e207a8307b591ab Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 14:11:46 +0430 Subject: [PATCH 107/109] removed more unused code --- jedi/common.py | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/jedi/common.py b/jedi/common.py index 04d17a5d..960d71d3 100644 --- a/jedi/common.py +++ b/jedi/common.py @@ -5,7 +5,6 @@ import functools import tokenizer as tokenize from jedi._compatibility import next, reraise -from jedi import debug from jedi import settings @@ -101,28 +100,7 @@ class NoErrorTokenizer(object): def __next__(self): if self.closed: raise MultiLevelStopIteration() - try: - self.current = next(self.gen) - except tokenize.TokenError: - # We just ignore this error, I try to handle it earlier - as - # good as possible - debug.warning('parentheses not closed error') - return self.__next__() - except IndentationError: - # This is an error, that tokenize may produce, because the code - # is not indented as it should. Here it just ignores this line - # and restarts the parser. - # (This is a rather unlikely error message, for normal code, - # tokenize seems to be pretty tolerant) - debug.warning('indentation error on line %s, ignoring it' % - self.current[2][0]) - # add the starting line of the last position - self.offset = (self.offset[0] + self.current[2][0], - self.current[2][1]) - self.gen = PushBackIterator(tokenize.generate_tokens( - self.readline)) - return self.__next__() - + self.current = next(self.gen) c = list(self.current) # stop if a new class or definition is started at position zero. From 8c56fba1e9946cc92d1accf8aeaba783e6582315 Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 14:12:15 +0430 Subject: [PATCH 108/109] fix a problem with statements without parents --- jedi/parsing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/jedi/parsing.py b/jedi/parsing.py index 2241b12d..e1785c01 100644 --- a/jedi/parsing.py +++ b/jedi/parsing.py @@ -394,6 +394,7 @@ class Parser(object): stmt = stmt_class(self.module, set_vars, used_vars, tok_list, first_pos, self.end_pos) + stmt.parent = self.top_module self._check_user_stmt(stmt) # Attribute docstring (PEP 257) support From d27dc0d58efccefa2440eb76e2c6b6ea19a7b52a Mon Sep 17 00:00:00 2001 From: David Halter Date: Sat, 20 Apr 2013 14:37:18 +0430 Subject: [PATCH 109/109] remove py2.5 from travis - this also removes python 2.5 support --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 16d38fab..29c03f1a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ language: python env: - - TOXENV=py25 PIP_INSECURE=t - TOXENV=py26 - TOXENV=py27 - TOXENV=py32