From a31ba8737abff01afbf169d3fb181dd2498ee5ad Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 06:46:31 +0100 Subject: [PATCH 01/25] Run refactoring test using py.test refactor.collect_file_tests is fixed; it uses global variable refactoring_test_dir which is not defined when refactor is used as a module. --- test/conftest.py | 11 ++++++++--- test/refactor.py | 7 +++---- test/test_integration.py | 11 +++++++++++ 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 3ed26cfb..ae57594f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,6 +2,7 @@ from os.path import join, dirname, abspath default_base_dir = join(dirname(abspath(__file__)), 'completion') import run +import refactor def pytest_addoption(parser): @@ -38,11 +39,15 @@ def pytest_generate_tests(metafunc): """ :type metafunc: _pytest.python.Metafunc """ + base_dir = metafunc.config.option.base_dir + test_files = dict(map(parse_test_files_option, + metafunc.config.option.test_files)) if 'case' in metafunc.fixturenames: - base_dir = metafunc.config.option.base_dir - test_files = dict(map(parse_test_files_option, - metafunc.config.option.test_files)) thirdparty = metafunc.config.option.thirdparty metafunc.parametrize( 'case', run.collect_dir_tests(base_dir, test_files, thirdparty)) + if 'refactor_case' in metafunc.fixturenames: + metafunc.parametrize( + 'refactor_case', + refactor.collect_dir_tests(base_dir, test_files)) diff --git a/test/refactor.py b/test/refactor.py index e4cc7c08..83bcaa98 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -64,7 +64,7 @@ class RefactoringCase(object): self.name, self.line_nr - 1) -def collect_file_tests(source, f_name, lines_to_execute): +def collect_file_tests(source, path, lines_to_execute): r = r'^# --- ?([^\n]*)\n((?:(?!\n# \+\+\+).)*)' \ r'\n# \+\+\+((?:(?!\n# ---).)*)' for match in re.finditer(r, source, re.DOTALL | re.MULTILINE): @@ -86,7 +86,6 @@ def collect_file_tests(source, f_name, lines_to_execute): if lines_to_execute and line_nr - 1 not in lines_to_execute: continue - path = os.path.join(os.path.abspath(refactoring_test_dir), f_name) yield RefactoringCase(name, source, line_nr, index, path, new_name, start_line_test, second) @@ -96,10 +95,10 @@ def collect_dir_tests(base_dir, test_files): files_to_execute = [a for a in test_files.items() if a[0] in f_name] lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, []) if f_name.endswith(".py") and (not test_files or files_to_execute): - path = os.path.join(refactoring_test_dir, f_name) + path = os.path.join(base_dir, f_name) with open(path) as f: source = f.read() - for case in collect_file_tests(source, f_name, lines_to_execute): + for case in collect_file_tests(source, path, lines_to_execute): yield case diff --git a/test/test_integration.py b/test/test_integration.py index 69ef90e1..c481fb5c 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -101,3 +101,14 @@ def test_integration(case, monkeypatch, pytestconfig): TEST_USAGES: run_related_name_test, } testers[case.test_type](case) + + +def test_refactor(refactor_case): + """ + Run refactoring test case. + + :type refactor_case: :class:`.refactor.RefactoringCase` + """ + refactor_case.run() + result, desired = refactor_case.result, refactor_case.desired + assert result == desired, "Refactoring test %r fails" % refactor_case From a993dd0da44cb4425be58bcf8b6b2c7eeb9212c7 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:00:44 +0100 Subject: [PATCH 02/25] Fix test_refactor It was not run because test cases were collected from test/complete instead of test/refactor. --- test/conftest.py | 14 ++++++++++---- test/test_integration.py | 3 ++- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index ae57594f..21f2e0e4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,14 +1,19 @@ -from os.path import join, dirname, abspath -default_base_dir = join(dirname(abspath(__file__)), 'completion') +import os +import base import run import refactor def pytest_addoption(parser): parser.addoption( - "--base-dir", default=default_base_dir, + "--integration-case-dir", + default=os.path.join(base.test_dir, 'completion'), help="Directory in which integration test case files locate.") + parser.addoption( + "--refactor-case-dir", + default=os.path.join(base.test_dir, 'refactor'), + help="Directory in which refactoring test case files locate.") parser.addoption( "--test-files", "-T", default=[], action='append', help=( @@ -39,15 +44,16 @@ def pytest_generate_tests(metafunc): """ :type metafunc: _pytest.python.Metafunc """ - base_dir = metafunc.config.option.base_dir test_files = dict(map(parse_test_files_option, metafunc.config.option.test_files)) if 'case' in metafunc.fixturenames: + base_dir = metafunc.config.option.integration_case_dir thirdparty = metafunc.config.option.thirdparty metafunc.parametrize( 'case', run.collect_dir_tests(base_dir, test_files, thirdparty)) if 'refactor_case' in metafunc.fixturenames: + base_dir = metafunc.config.option.refactor_case_dir metafunc.parametrize( 'refactor_case', refactor.collect_dir_tests(base_dir, test_files)) diff --git a/test/test_integration.py b/test/test_integration.py index c481fb5c..fe09013a 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,6 +1,7 @@ import os import re +import base from run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES @@ -92,7 +93,7 @@ def run_related_name_test(case): def test_integration(case, monkeypatch, pytestconfig): - repo_root = os.path.dirname(os.path.dirname(pytestconfig.option.base_dir)) + repo_root = base.root_dir monkeypatch.chdir(os.path.join(repo_root, 'jedi')) testers = { TEST_COMPLETIONS: run_completion_test, From 51a094be028d4275d1265792158a907506ccdecf Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:05:30 +0100 Subject: [PATCH 03/25] Run py.test in tox.ini --- pytest.ini | 2 ++ tox.ini | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 pytest.ini diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..2ce9da86 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --assert=plain diff --git a/tox.ini b/tox.ini index 2a39e11c..afcc4cbf 100644 --- a/tox.ini +++ b/tox.ini @@ -4,13 +4,12 @@ envlist = py25, py26, py27, py32 setenv = XDG_CACHE_HOME={envtmpdir}/cache deps = - nose + pytest commands = - python regression.py - python run.py - python refactor.py - nosetests --with-doctest --doctest-tests {toxinidir}/jedi -changedir = test + py.test [] + # Doctests can't be run with the main tests because then py.test + # tries to import broken python files under test/*/. + py.test --doctest-modules {toxinidir}/jedi [testenv:py25] deps = simplejson From 0f9761aac73d3c6de849b83280944437d7965d25 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:11:37 +0100 Subject: [PATCH 04/25] Fix tests for Python 3.2 (use relative import) --- test/conftest.py | 6 +++--- test/refactor.py | 2 +- test/regression.py | 2 +- test/run.py | 2 +- test/test_integration.py | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 21f2e0e4..8d94cbd3 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,8 +1,8 @@ import os -import base -import run -import refactor +from . import base +from . import run +from . import refactor def pytest_addoption(parser): diff --git a/test/refactor.py b/test/refactor.py index 83bcaa98..058af2a2 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -10,7 +10,7 @@ import traceback import re import itertools -import base +from . import base from jedi._compatibility import reduce import jedi diff --git a/test/regression.py b/test/regression.py index 54dfe755..b327811c 100755 --- a/test/regression.py +++ b/test/regression.py @@ -11,7 +11,7 @@ import itertools import os import textwrap -from base import TestBase, unittest, cwd_at +from .base import TestBase, unittest, cwd_at import jedi from jedi._compatibility import is_py25, utf8, unicode diff --git a/test/run.py b/test/run.py index 106f6359..2692439f 100755 --- a/test/run.py +++ b/test/run.py @@ -54,7 +54,7 @@ import re import traceback import itertools -import base +from . import base from jedi._compatibility import unicode, StringIO, reduce, literal_eval, is_py25 diff --git a/test/test_integration.py b/test/test_integration.py index fe09013a..0e408c4f 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,8 +1,8 @@ import os import re -import base -from run import \ +from . import base +from .run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES import jedi From 180d0a87648959cb082d51ad2234526048374f1a Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:12:44 +0100 Subject: [PATCH 05/25] Rename regression.py to test_regression.py in order to let py.test collect the tests. --- test/{regression.py => test_regression.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/{regression.py => test_regression.py} (100%) diff --git a/test/regression.py b/test/test_regression.py similarity index 100% rename from test/regression.py rename to test/test_regression.py From 71bb93224d82b1eabebe08076144266e0afd321c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:48:20 +0100 Subject: [PATCH 06/25] Ignore first N failures in Python 2.5 --- test/base.py | 32 ++++++++++++++++++++++++++++++++ test/test_integration.py | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/test/base.py b/test/base.py index a9e2d9ab..4277bb71 100644 --- a/test/base.py +++ b/test/base.py @@ -12,8 +12,11 @@ test_dir = dirname(abspath(__file__)) root_dir = dirname(test_dir) sys.path.insert(0, root_dir) +import pytest + import jedi from jedi import debug +from jedi._compatibility import is_py25 test_sum = 0 t_start = time.time() @@ -102,3 +105,32 @@ def cwd_at(path): os.chdir(oldcwd) return wrapper return decorator + + +_py25_fails = 0 +py25_allowed_fails = 9 + + +def skip_py25_fails(func): + """ + Skip first `py25_allowed_fails` failures in Python 2.5. + + .. todo:: Remove this decorator by implementing "skip tag" for + integration tests. + """ + @functools.wraps(func) + def wrapper(*args, **kwds): + global _py25_fails + try: + func(*args, **kwds) + except AssertionError: + _py25_fails += 1 + if _py25_fails > py25_allowed_fails: + raise + else: + pytest.skip("%d-th failure (there can be %d failures)" % + (_py25_fails, py25_allowed_fails)) + return wrapper + +if not is_py25: + skip_py25_fails = lambda f: f diff --git a/test/test_integration.py b/test/test_integration.py index 0e408c4f..f693b31b 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -101,7 +101,7 @@ def test_integration(case, monkeypatch, pytestconfig): TEST_ASSIGNMENTS: run_goto_test, TEST_USAGES: run_related_name_test, } - testers[case.test_type](case) + base.skip_py25_fails(testers[case.test_type])(case) def test_refactor(refactor_case): From 13b48632e599ab5e56f394e78c0f81a608af3af9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 07:58:40 +0100 Subject: [PATCH 07/25] Better assertion message formatter --- test/test_integration.py | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/test/test_integration.py b/test/test_integration.py index f693b31b..f348689d 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -9,16 +9,28 @@ import jedi from jedi._compatibility import literal_eval +def assert_case_equal(case, actual, desired): + """ + Assert ``actual == desired`` with formatted message. + + This is not needed for typical py.test use case, but as we need + ``--assert=plain`` (see ../pytest.ini) to workaround some issue + due to py.test magic, let's format the message by hand. + """ + assert actual == desired, """ +Test %r failed. +actual = %s +desired = %s +""" % (case, actual, desired) + + def run_completion_test(case): (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) completions = script.complete() #import cProfile; cProfile.run('script.complete()') comp_str = set([c.word for c in completions]) - if comp_str != set(literal_eval(correct)): - raise AssertionError( - 'Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, comp_str, correct)) + assert_case_equal(case, comp_str, set(literal_eval(correct))) def run_definition_test(case): @@ -53,19 +65,14 @@ def run_definition_test(case): should_str = definition(correct, start, script.source_path) result = script.definition() is_str = set(r.desc_with_module for r in result) - if is_str != should_str: - raise AssertionError( - 'Solution @%s not right, received %s, wanted %s' - % (line_nr - 1, is_str, should_str)) + assert_case_equal(case, is_str, should_str) def run_goto_test(case): (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) result = script.goto() comp_str = str(sorted(str(r.description) for r in result)) - if comp_str != correct: - raise AssertionError('Solution @%s not right, received %s, wanted %s' - % (line_nr - 1, comp_str, correct)) + assert_case_equal(case, comp_str, correct) def run_related_name_test(case): @@ -86,10 +93,7 @@ def run_related_name_test(case): else: wanted.append(('renaming', line_nr + pos_tup[0], pos_tup[1])) - wanted = sorted(wanted) - if compare != wanted: - raise AssertionError('Solution @%s not right, received %s, wanted %s' - % (line_nr - 1, compare, wanted)) + assert_case_equal(case, compare, sorted(wanted)) def test_integration(case, monkeypatch, pytestconfig): @@ -111,5 +115,5 @@ def test_refactor(refactor_case): :type refactor_case: :class:`.refactor.RefactoringCase` """ refactor_case.run() - result, desired = refactor_case.result, refactor_case.desired - assert result == desired, "Refactoring test %r fails" % refactor_case + assert_case_equal(refactor_case, + refactor_case.result, refactor_case.desired) From 88adcbcf8ac77f0c90b0a90147a06264ffbe2151 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 08:03:55 +0100 Subject: [PATCH 08/25] Use tox in .travis.yml --- .travis.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index ba8fb069..81b0cea4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,13 +5,9 @@ python: - 2.7 - 3.2 install: - - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - pip install --use-mirrors simplejson unittest2; - fi - - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then - pip install --use-mirrors unittest2; - fi - - pip install --use-mirrors nose + - pip install --use-mirrors tox script: - - cd test - - ./test.sh + - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | + sed --regexp-extended 's/([0-9])\.([0-9])/py\1\2/g') + - echo "TOXENV=$TOXENV" + - tox From 674743b7bbb71a2bf4003dd38daa224fa8dcaf56 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 08:49:24 +0100 Subject: [PATCH 09/25] Avoid NoSSLError in Python 2.5 --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index afcc4cbf..bc94b159 100644 --- a/tox.ini +++ b/tox.ini @@ -14,6 +14,7 @@ commands = deps = simplejson unittest2 + ssl {[testenv]deps} [testenv:py26] deps = From e7b352b82617f392728e4fd4502311bfedefbeab Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:01:56 +0100 Subject: [PATCH 10/25] Install ssl in .travis.yml --- .travis.yml | 3 +++ tox.ini | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 81b0cea4..3db7151f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,6 +5,9 @@ python: - 2.7 - 3.2 install: + - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then + pip install --use-mirrors ssl; + fi - pip install --use-mirrors tox script: - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | diff --git a/tox.ini b/tox.ini index bc94b159..afcc4cbf 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,6 @@ commands = deps = simplejson unittest2 - ssl {[testenv]deps} [testenv:py26] deps = From 9e600ed0b15bd82b9ffa30c1c1e64dd14dbd56ec Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:04:56 +0100 Subject: [PATCH 11/25] Use --insecure when running pip with Python 2.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3db7151f..77b753ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: - 3.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - pip install --use-mirrors ssl; + pip install --use-mirrors --insecure ssl; fi - pip install --use-mirrors tox script: From 135dd56e61bd97f4c30197bd9c49c4fcbc0bf0d8 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:15:30 +0100 Subject: [PATCH 12/25] Install libssl-dev for Python 2.5 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 77b753ea..01e5be0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,8 @@ python: - 3.2 install: - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - pip install --use-mirrors --insecure ssl; + sudo apt-get install libssl-dev && + pip install --use-mirrors ssl; fi - pip install --use-mirrors tox script: From ab33400f76486d389319ba3f1385973002ea11f6 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:18:40 +0100 Subject: [PATCH 13/25] Install libbluetooth-dev for Python 2.5 --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 01e5be0b..294f1503 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,9 @@ python: - 2.7 - 3.2 install: + # See: http://stackoverflow.com/questions/3241658/ - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - sudo apt-get install libssl-dev && + sudo apt-get install libbluetooth-dev && pip install --use-mirrors ssl; fi - pip install --use-mirrors tox From 5c3252908fc42a2887e3dcdda339d9ca260f6f56 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:29:00 +0100 Subject: [PATCH 14/25] Use PIP_INSECURE=t --- .travis.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 294f1503..7a5adc19 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,11 +5,8 @@ python: - 2.7 - 3.2 install: - # See: http://stackoverflow.com/questions/3241658/ - - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then - sudo apt-get install libbluetooth-dev && - pip install --use-mirrors ssl; - fi + # Needed only for Python 2.5: + - export PIP_INSECURE=t - pip install --use-mirrors tox script: - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" | From cee167e3d29c951e51894291eb65ec700082a874 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 08:50:32 +0100 Subject: [PATCH 15/25] Run py.test in clean cache directory And finally remove XDG_CACHE_HOME=... in tox.ini. --- pytest.ini | 5 +++++ test/conftest.py | 26 ++++++++++++++++++++++++++ test/test_integration.py | 2 ++ tox.ini | 2 -- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 2ce9da86..4afbf164 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,7 @@ [pytest] addopts = --assert=plain + +# Activate `clean_jedi_cache` fixture for all tests. This should be +# fine as long as we are using `clean_jedi_cache` as a session scoped +# fixture. +usefixtures = clean_jedi_cache diff --git a/test/conftest.py b/test/conftest.py index 8d94cbd3..4cb2f4e5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,8 @@ import os +import shutil +import tempfile + +import pytest from . import base from . import run @@ -57,3 +61,25 @@ def pytest_generate_tests(metafunc): metafunc.parametrize( 'refactor_case', refactor.collect_dir_tests(base_dir, test_files)) + + +@pytest.fixture(scope='session') +def clean_jedi_cache(request): + """ + Set `jedi.settings.cache_directory` to a temporary directory during test. + + Note that you can't use built-in `tmpdir` and `monkeypatch` + fixture here because their scope is 'function', which is not used + in 'session' scope fixture. + + This fixture is activated in ../pytest.ini. + """ + settings = base.jedi.settings + old = settings.cache_directory + tmp = tempfile.mkdtemp(prefix='jedi-test-') + settings.cache_directory = tmp + + @request.addfinalizer + def restore(): + settings.cache_directory = old + shutil.rmtree(tmp) diff --git a/test/test_integration.py b/test/test_integration.py index f348689d..b3baf336 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,6 +1,8 @@ import os import re +import pytest + from . import base from .run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES diff --git a/tox.ini b/tox.ini index afcc4cbf..55022b23 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,6 @@ [tox] envlist = py25, py26, py27, py32 [testenv] -setenv = - XDG_CACHE_HOME={envtmpdir}/cache deps = pytest commands = From 7c289ce6be981eaf02f71dd11135a3a30e765235 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:52:58 +0100 Subject: [PATCH 16/25] Workaround test failure due to cache in Python 3.2 --- tox.ini | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tox.ini b/tox.ini index 55022b23..9ab832ab 100644 --- a/tox.ini +++ b/tox.ini @@ -17,3 +17,8 @@ deps = deps = unittest2 {[testenv]deps} +[testenv:py32] +# TODO: Without this setting, test uses ~/.cache/jedi/. +# There could be a bug due to import hack. +setenv = + XDG_CACHE_HOME={envtmpdir}/cache From 49f635dca3f8e8e6978f4a056c90184174f46257 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:01:57 +0100 Subject: [PATCH 17/25] Add a failing test due to import hack --- test/test_regression.py | 13 +++++++++++++ tox.ini | 1 + 2 files changed, 14 insertions(+) diff --git a/test/test_regression.py b/test/test_regression.py index b327811c..2e0da3cf 100755 --- a/test/test_regression.py +++ b/test/test_regression.py @@ -11,6 +11,8 @@ import itertools import os import textwrap +import pytest + from .base import TestBase, unittest, cwd_at import jedi @@ -524,5 +526,16 @@ class TestSpeed(TestBase): script.function_definition() #print(jedi.imports.imports_processed) + +@pytest.mark.skipif("sys.version_info >= (3,0)") +def test_settings_module(): + """ + jedi.settings and jedi.cache.settings must be the same module. + """ + from jedi import cache + from jedi import settings + assert cache.settings is settings + + if __name__ == '__main__': unittest.main() diff --git a/tox.ini b/tox.ini index 9ab832ab..25014940 100644 --- a/tox.ini +++ b/tox.ini @@ -20,5 +20,6 @@ deps = [testenv:py32] # TODO: Without this setting, test uses ~/.cache/jedi/. # There could be a bug due to import hack. +# See test_settings_module in test/test_regression.py. setenv = XDG_CACHE_HOME={envtmpdir}/cache From ff80988a75b6f1bcf6ecfff5ef837d2e02e8ef3c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 09:39:58 +0100 Subject: [PATCH 18/25] Remove old test code --- test/refactor.py | 60 +---------- test/run.py | 269 ++++++----------------------------------------- test/test.sh | 8 -- 3 files changed, 35 insertions(+), 302 deletions(-) delete mode 100755 test/test.sh diff --git a/test/refactor.py b/test/refactor.py index 058af2a2..206e1662 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -10,8 +10,7 @@ import traceback import re import itertools -from . import base - +from . import base # required to setup import path from jedi._compatibility import reduce import jedi from jedi import refactoring @@ -100,60 +99,3 @@ def collect_dir_tests(base_dir, test_files): source = f.read() for case in collect_file_tests(source, path, lines_to_execute): yield case - - -def run_test(cases): - """ - This is the completion test for some cases. The tests are not unit test - like, they are rather integration tests. - It uses comments to specify a test in the next line. The comment also says, - which results are expected. The comment always begins with `#?`. The last - row symbolizes the cursor. - - For example:: - - #? ['ab'] - ab = 3; a - - #? int() - ab = 3; ab - """ - fails = 0 - tests = 0 - for case in cases: - try: - if not case.check(): - print(case) - print(' ' + repr(str(case.result))) - print(' ' + repr(case.desired)) - fails += 1 - except Exception: - print(traceback.format_exc()) - print(case) - fails += 1 - tests += 1 - return tests, fails - - -def test_dir(refactoring_test_dir): - for (path, cases) in itertools.groupby( - collect_dir_tests(refactoring_test_dir, test_files), - lambda case: case.path): - num_tests, fails = run_test(cases) - - base.test_sum += num_tests - f_name = os.path.basename(path) - s = 'run %s tests with %s fails (%s)' % (num_tests, fails, f_name) - base.tests_fail += fails - print(s) - base.summary.append(s) - - -if __name__ == '__main__': - refactoring_test_dir = os.path.join(base.test_dir, 'refactor') - test_files = base.get_test_list() - test_dir(refactoring_test_dir) - - base.print_summary() - - sys.exit(1 if base.tests_fail else 0) diff --git a/test/run.py b/test/run.py index 2692439f..10cdf26d 100755 --- a/test/run.py +++ b/test/run.py @@ -31,35 +31,58 @@ If you want to debug a test, just use the --debug option. Auto-Completion +++++++++++++++ -.. autofunction:: run_completion_test +Uses comments to specify a test in the next line. The comment says, which +results are expected. The comment always begins with `#?`. The last row +symbolizes the cursor. + +For example:: + + #? ['real'] + a = 3; a.rea + +Because it follows ``a.rea`` and a is an ``int``, which has a ``real`` +property. Definition ++++++++++ -.. autofunction:: run_definition_test +Definition tests use the same symbols like completion tests. This is +possible because the completion tests are defined with a list:: + + #? int() + ab = 3; ab Goto ++++ -.. autofunction:: run_goto_test +Tests look like this:: + + abc = 1 + #! ['abc=1'] + abc + +Additionally it is possible to add a number which describes to position of +the test (otherwise it's just end of line):: + + #! 2 ['abc=1'] + abc Related Names +++++++++++++ -.. autofunction:: run_related_name_test +Tests look like this:: + + abc = 1 + #< abc@1,0 abc@3,0 + abc """ import os import sys import re -import traceback -import itertools - -from . import base - -from jedi._compatibility import unicode, StringIO, reduce, literal_eval, is_py25 +from . import base # required to setup import path import jedi -from jedi import debug +from jedi._compatibility import unicode, StringIO, reduce, is_py25 sys.path.pop(0) # pop again, because it might affect the completion @@ -71,147 +94,6 @@ TEST_ASSIGNMENTS = 2 TEST_USAGES = 3 -def run_completion_test(case): - """ - Uses comments to specify a test in the next line. The comment says, which - results are expected. The comment always begins with `#?`. The last row - symbolizes the cursor. - - For example:: - - #? ['real'] - a = 3; a.rea - - Because it follows ``a.rea`` and a is an ``int``, which has a ``real`` - property. - - Returns 1 for fail and 0 for success. - """ - (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) - completions = script.complete() - #import cProfile; cProfile.run('script.complete()') - - comp_str = set([c.word for c in completions]) - if comp_str != set(literal_eval(correct)): - print('Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, comp_str, correct)) - return 1 - return 0 - - -def run_definition_test(case): - """ - Definition tests use the same symbols like completion tests. This is - possible because the completion tests are defined with a list:: - - #? int() - ab = 3; ab - - Returns 1 for fail and 0 for success. - """ - def definition(correct, correct_start, path): - def defs(line_nr, indent): - s = jedi.Script(script.source, line_nr, indent, path) - return set(s.definition()) - - should_be = set() - number = 0 - for index in re.finditer('(?: +|$)', correct): - if correct == ' ': - continue - # -1 for the comment, +3 because of the comment start `#? ` - start = index.start() - if base.print_debug: - jedi.set_debug_function(None) - number += 1 - try: - should_be |= defs(line_nr - 1, start + correct_start) - except Exception: - print('could not resolve %s indent %s' % (line_nr - 1, start)) - raise - if base.print_debug: - jedi.set_debug_function(debug.print_to_stdout) - # because the objects have different ids, `repr` it, then compare it. - should_str = set(r.desc_with_module for r in should_be) - if len(should_str) < number: - raise Exception('Solution @%s not right, too few test results: %s' - % (line_nr - 1, should_str)) - return should_str - - (correct, line_nr, column, start, line) = \ - (case.correct, case.line_nr, case.column, case.start, case.line) - script = case.script() - should_str = definition(correct, start, script.source_path) - result = script.definition() - is_str = set(r.desc_with_module for r in result) - if is_str != should_str: - print('Solution @%s not right, received %s, wanted %s' \ - % (line_nr - 1, is_str, should_str)) - return 1 - return 0 - - -def run_goto_test(case): - """ - Tests look like this:: - - abc = 1 - #! ['abc=1'] - abc - - Additionally it is possible to add a number which describes to position of - the test (otherwise it's just end of line):: - - #! 2 ['abc=1'] - abc - - Returns 1 for fail and 0 for success. - """ - (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) - result = script.goto() - comp_str = str(sorted(str(r.description) for r in result)) - if comp_str != correct: - print('Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, comp_str, correct)) - return 1 - return 0 - - -def run_related_name_test(case): - """ - Tests look like this:: - - abc = 1 - #< abc@1,0 abc@3,0 - abc - - Returns 1 for fail and 0 for success. - """ - (script, correct, line_nr) = (case.script(), case.correct, case.line_nr) - result = script.related_names() - correct = correct.strip() - compare = sorted((r.module_name, r.start_pos[0], r.start_pos[1]) - for r in result) - wanted = [] - if not correct: - positions = [] - else: - positions = literal_eval(correct) - for pos_tup in positions: - if type(pos_tup[0]) == str: - # this means that there is a module specified - wanted.append(pos_tup) - else: - wanted.append(('renaming', line_nr + pos_tup[0], pos_tup[1])) - - wanted = sorted(wanted) - if compare != wanted: - print('Solution @%s not right, received %s, wanted %s'\ - % (line_nr - 1, compare, wanted)) - return 1 - return 0 - - class IntegrationTestCase(object): def __init__(self, test_type, correct, line_nr, column, start, line, @@ -290,86 +172,3 @@ def collect_dir_tests(base_dir, test_files, thirdparty=False): case.path = path case.source = source yield case - - -def run_test(cases): - """ - This is the completion test for some cases. The tests are not unit test - like, they are rather integration tests. - """ - testers = { - TEST_COMPLETIONS: run_completion_test, - TEST_DEFINITIONS: run_definition_test, - TEST_ASSIGNMENTS: run_goto_test, - TEST_USAGES: run_related_name_test, - } - - tests = 0 - fails = 0 - for case in cases: - tests += 1 - try: - fails += testers[case.test_type](case) - except Exception: - print(traceback.format_exc()) - print(case) - fails += 1 - return tests, fails - - -def test_dir(completion_test_dir, thirdparty=False): - for (path, cases) in itertools.groupby( - collect_dir_tests(completion_test_dir, test_files, thirdparty), - lambda case: case.path): - f_name = os.path.basename(path) - - if thirdparty: - lib = f_name.replace('_.py', '') - try: - # there is always an underline at the end. - # It looks like: completion/thirdparty/pylab_.py - __import__(lib) - except ImportError: - base.summary.append('Thirdparty-Library %s not found.' % - f_name) - continue - - num_tests, fails = run_test(cases) - base.test_sum += num_tests - - s = 'run %s tests with %s fails (%s)' % (num_tests, fails, f_name) - base.tests_fail += fails - print(s) - base.summary.append(s) - - -if __name__ == '__main__': - try: - i = sys.argv.index('--thirdparty') - thirdparty = True - sys.argv = sys.argv[:i] + sys.argv[i + 1:] - except ValueError: - thirdparty = False - - test_files = base.get_test_list() - - # completion tests: - completion_test_dir = os.path.join(base.test_dir, 'completion') - - # execute tests - test_dir(completion_test_dir) - if test_files or thirdparty: - completion_test_dir += '/thirdparty' - test_dir(completion_test_dir, thirdparty=True) - - base.print_summary() - #from guppy import hpy - #hpy() - #print hpy().heap() - - exit_code = 1 if base.tests_fail else 0 - if sys.hexversion < 0x02060000 and base.tests_fail <= 9: - # Python 2.5 has major incompabillities (e.g. no property.setter), - # therefore it is not possible to pass all tests. - exit_code = 0 - sys.exit(exit_code) diff --git a/test/test.sh b/test/test.sh deleted file mode 100755 index 4b41f743..00000000 --- a/test/test.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -e - -python regression.py -python run.py -echo -python refactor.py -echo -nosetests --with-doctest --doctest-tests ../jedi/ From b5764c14461e1704daefaf8d9137ddcb3590dea9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:04:06 +0100 Subject: [PATCH 19/25] Remove old utility functions in test.base --- test/base.py | 45 --------------------------------------------- 1 file changed, 45 deletions(-) diff --git a/test/base.py b/test/base.py index 4277bb71..f01ca5cd 100644 --- a/test/base.py +++ b/test/base.py @@ -1,4 +1,3 @@ -import time import sys if sys.hexversion < 0x02070000: import unittest2 as unittest @@ -15,45 +14,8 @@ sys.path.insert(0, root_dir) import pytest import jedi -from jedi import debug from jedi._compatibility import is_py25 -test_sum = 0 -t_start = time.time() -# Sorry I didn't use argparse here. It's because argparse is not in the -# stdlib in 2.5. -args = sys.argv[1:] - -print_debug = False -try: - i = args.index('--debug') - args = args[:i] + args[i + 1:] -except ValueError: - pass -else: - print_debug = True - jedi.set_debug_function(debug.print_to_stdout) - -sys.argv = sys.argv[:1] + args - -summary = [] -tests_fail = 0 - - -def get_test_list(): -# get test list, that should be executed - test_files = {} - last = None - for arg in sys.argv[1:]: - if arg.isdigit(): - if last is None: - continue - test_files[last].append(int(arg)) - else: - test_files[arg] = [] - last = arg - return test_files - class TestBase(unittest.TestCase): def get_script(self, src, pos, path=None): @@ -79,13 +41,6 @@ class TestBase(unittest.TestCase): return script.function_definition() -def print_summary(): - print('\nSummary: (%s fails of %s tests) in %.3fs' % \ - (tests_fail, test_sum, time.time() - t_start)) - for s in summary: - print(s) - - def cwd_at(path): """ Decorator to run function at `path`. From 1fffbf13cae48954b8af4dc2f5c53afd0b18eafd Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:14:25 +0100 Subject: [PATCH 20/25] Fix test failures because imports.py uses base.py --- test/base.py | 3 +++ test/completion/imports.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/base.py b/test/base.py index f01ca5cd..a0e6666a 100644 --- a/test/base.py +++ b/test/base.py @@ -17,6 +17,9 @@ import jedi from jedi._compatibility import is_py25 +sample_int = 1 # This is used in completion/imports.py + + class TestBase(unittest.TestCase): def get_script(self, src, pos, path=None): if pos is None: diff --git a/test/completion/imports.py b/test/completion/imports.py index af73b99c..e651d97e 100644 --- a/test/completion/imports.py +++ b/test/completion/imports.py @@ -154,9 +154,9 @@ mod1.a from .. import base #? int() -base.tests_fail +base.sample_int -from ..base import tests_fail as f +from ..base import sample_int as f #? int() f From 446c7cf6943659d7e67db19d0084202cc0ad95e3 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 10:36:42 +0100 Subject: [PATCH 21/25] Document how to run test --- test/run.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/test/run.py b/test/run.py index 10cdf26d..ec2ac523 100755 --- a/test/run.py +++ b/test/run.py @@ -17,16 +17,36 @@ There are different kind of tests: How to run tests? +++++++++++++++++ -Basically ``run.py`` searches the ``completion`` directory for files with lines -starting with the symbol above. There is also support for third party -libraries. In a normal test run (``./run.py``) they are not being executed, you -have to provide a ``--thirdparty`` option. +Jedi uses pytest_ to run unit and integration tests. To run tests, +simply run ``py.test``. You can also use tox_ to run tests for +multiple Python versions. -Now it's much more important, that you know how test only one file (``./run.py -classes``, where ``classes`` is the name of the file to test) or even one test -(``./run.py classes 90``, which would just execute the test on line 90). +.. _pytest: http://pytest.org +.. _tox: http://testrun.org/tox -If you want to debug a test, just use the --debug option. +Integration test cases are located in ``test/completion`` directory +and each test cases are indicated by the comment ``#?`` (complete / +definitions), ``#!`` (assignments) and ``#<`` (usages). There is also +support for third party libraries. In a normal test run they are not +being executed, you have to provide a ``--thirdparty`` option. + +In addition to standard `-k` and `-m` options in py.test, you can use +`-T` (`--test-files`) option to specify integration test cases to run. +It takes the format of ``FILE_NAME[:LINE[,LINE[,...]]]`` where +``FILE_NAME`` is a file in ``test/completion`` and ``LINE`` is a line +number of the test comment. Here is some recipes: + +Run tests only in ``basic.py`` and ``imports.py``:: + + py.test test/test_integration.py -T basic.py -T imports.py + +Run test at line 4, 6, and 8 in ``basic.py``:: + + py.test test/test_integration.py -T basic.py:4,6,8 + +See ``py.test --help`` for more information. + +If you want to debug a test, just use the --pdb option. Auto-Completion +++++++++++++++ From c387bf06bcc72c4b4af7a2a24baa80a273c7d8d9 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:03:12 +0100 Subject: [PATCH 22/25] No need to setup import path in test.base py.test does that for us. --- test/base.py | 1 - test/refactor.py | 1 - test/run.py | 5 ----- 3 files changed, 7 deletions(-) diff --git a/test/base.py b/test/base.py index a0e6666a..efdd9f18 100644 --- a/test/base.py +++ b/test/base.py @@ -9,7 +9,6 @@ import functools test_dir = dirname(abspath(__file__)) root_dir = dirname(test_dir) -sys.path.insert(0, root_dir) import pytest diff --git a/test/refactor.py b/test/refactor.py index 206e1662..05a8d154 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -10,7 +10,6 @@ import traceback import re import itertools -from . import base # required to setup import path from jedi._compatibility import reduce import jedi from jedi import refactoring diff --git a/test/run.py b/test/run.py index ec2ac523..e37abab2 100755 --- a/test/run.py +++ b/test/run.py @@ -97,17 +97,12 @@ Tests look like this:: abc """ import os -import sys import re -from . import base # required to setup import path import jedi from jedi._compatibility import unicode, StringIO, reduce, is_py25 -sys.path.pop(0) # pop again, because it might affect the completion - - TEST_COMPLETIONS = 0 TEST_DEFINITIONS = 1 TEST_ASSIGNMENTS = 2 From c87d3dad52d5beb65afae129a68dc8e9cf34bf3c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:04:15 +0100 Subject: [PATCH 23/25] Remove unused imports --- test/base.py | 7 ++++--- test/refactor.py | 3 --- test/test_integration.py | 2 -- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/test/base.py b/test/base.py index efdd9f18..2deb4452 100644 --- a/test/base.py +++ b/test/base.py @@ -7,15 +7,16 @@ import os from os.path import abspath, dirname import functools -test_dir = dirname(abspath(__file__)) -root_dir = dirname(test_dir) - import pytest import jedi from jedi._compatibility import is_py25 +test_dir = dirname(abspath(__file__)) +root_dir = dirname(test_dir) + + sample_int = 1 # This is used in completion/imports.py diff --git a/test/refactor.py b/test/refactor.py index 05a8d154..fdbbdc20 100755 --- a/test/refactor.py +++ b/test/refactor.py @@ -4,11 +4,8 @@ Refactoring tests work a little bit similar to Black Box tests. But the idea is here to compare two versions of code. """ from __future__ import with_statement -import sys import os -import traceback import re -import itertools from jedi._compatibility import reduce import jedi diff --git a/test/test_integration.py b/test/test_integration.py index b3baf336..f348689d 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -1,8 +1,6 @@ import os import re -import pytest - from . import base from .run import \ TEST_COMPLETIONS, TEST_DEFINITIONS, TEST_ASSIGNMENTS, TEST_USAGES From de7092d56b5232c29b6b10e6d3df1098e2ee8f8c Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:24:48 +0100 Subject: [PATCH 24/25] Fix: --thirdparty was not considered --- test/conftest.py | 10 ++++++---- test/run.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/test/conftest.py b/test/conftest.py index 4cb2f4e5..87a7fa9f 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -25,7 +25,7 @@ def pytest_addoption(parser): "For example: -T generators.py:10,13,19. " "Note that you can use -m to specify the test case by id.")) parser.addoption( - "--thirdparty", + "--thirdparty", action='store_true', help="Include integration tests that requires third party modules.") @@ -53,9 +53,11 @@ def pytest_generate_tests(metafunc): if 'case' in metafunc.fixturenames: base_dir = metafunc.config.option.integration_case_dir thirdparty = metafunc.config.option.thirdparty - metafunc.parametrize( - 'case', - run.collect_dir_tests(base_dir, test_files, thirdparty)) + cases = list(run.collect_dir_tests(base_dir, test_files)) + if thirdparty: + cases.extend(run.collect_dir_tests( + os.path.join(base_dir, 'thirdparty'), test_files)) + metafunc.parametrize('case', cases) if 'refactor_case' in metafunc.fixturenames: base_dir = metafunc.config.option.refactor_case_dir metafunc.parametrize( diff --git a/test/run.py b/test/run.py index e37abab2..16d08f8a 100755 --- a/test/run.py +++ b/test/run.py @@ -171,7 +171,7 @@ def collect_file_tests(lines, lines_to_execute): correct = None -def collect_dir_tests(base_dir, test_files, thirdparty=False): +def collect_dir_tests(base_dir, test_files): for f_name in os.listdir(base_dir): files_to_execute = [a for a in test_files.items() if a[0] in f_name] lines_to_execute = reduce(lambda x, y: x + y[1], files_to_execute, []) From e5b660b0a1d92cc9ce8e3ebf439f782fc6517108 Mon Sep 17 00:00:00 2001 From: Takafumi Arakaki Date: Tue, 12 Mar 2013 11:33:01 +0100 Subject: [PATCH 25/25] Use PIP_INSECURE=t only in Python 2.5 test --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7a5adc19..506376d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,9 @@ python: - 2.7 - 3.2 install: - # Needed only for Python 2.5: - - export PIP_INSECURE=t + - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then + export PIP_INSECURE=t; + fi - pip install --use-mirrors tox script: - export TOXENV=$(echo "$TRAVIS_PYTHON_VERSION" |