From ceccbf367850cfe256cd1921414e414a3a3b3d35 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 30 Jan 2020 19:18:41 +0100 Subject: [PATCH 01/20] Make the Project API public, fixes #778 --- jedi/__init__.py | 1 + jedi/api/__init__.py | 11 ++++++++--- test/test_api/test_usages.py | 2 +- test/test_inference/test_gradual/test_conversion.py | 8 ++++---- .../test_inference/test_gradual/test_stub_loading.py | 2 +- test/test_inference/test_gradual/test_stubs.py | 2 +- test/test_inference/test_imports.py | 12 ++++++------ 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/jedi/__init__.py b/jedi/__init__.py index efec776d..e1356feb 100644 --- a/jedi/__init__.py +++ b/jedi/__init__.py @@ -41,6 +41,7 @@ from jedi import settings from jedi.api.environment import find_virtualenvs, find_system_environments, \ get_default_environment, InvalidPythonEnvironment, create_environment, \ get_system_environment +from jedi.api.project import Project from jedi.api.exceptions import InternalError # Finally load the internal plugins. This is only internal. from jedi.plugins import registry diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 44751109..39f1c904 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -87,7 +87,7 @@ class Script(object): """ def __init__(self, source=None, line=None, column=None, path=None, encoding='utf-8', sys_path=None, environment=None, - _project=None): + project=None): self._orig_path = path # An empty path (also empty string) should always result in no path. self.path = os.path.abspath(path) if path else None @@ -103,7 +103,6 @@ class Script(object): if sys_path is not None and not is_py3: sys_path = list(map(force_unicode, sys_path)) - project = _project if project is None: # Load the Python grammar of the current interpreter. project = get_default_project( @@ -112,6 +111,12 @@ class Script(object): # TODO deprecate and remove sys_path from the Script API. if sys_path is not None: project._sys_path = sys_path + warnings.warn( + "Deprecated since version 0.17.0. Use the project API instead, " + "which means Script(project=Project(dir, sys_path=sys_path)) instead.", + DeprecationWarning, + stacklevel=2 + ) self._inference_state = InferenceState( project, environment=environment, script_path=self.path ) @@ -559,7 +564,7 @@ class Interpreter(Script): raise TypeError("The environment needs to be an InterpreterEnvironment subclass.") super(Interpreter, self).__init__(source, environment=environment, - _project=Project(os.getcwd()), **kwds) + project=Project(os.getcwd()), **kwds) self.namespaces = namespaces self._inference_state.allow_descriptor_getattr = self._allow_descriptor_getattr_default diff --git a/test/test_api/test_usages.py b/test/test_api/test_usages.py index 4d0effdc..188f893d 100644 --- a/test/test_api/test_usages.py +++ b/test/test_api/test_usages.py @@ -6,7 +6,7 @@ def test_import_references(Script): def test_exclude_builtin_modules(Script): def get(include): from jedi.api.project import Project - script = Script(source, _project=Project('', sys_path=[], smart_sys_path=False)) + script = Script(source, project=Project('', sys_path=[], smart_sys_path=False)) references = script.get_references(column=8, include_builtins=include) return [(d.line, d.column) for d in references] source = '''import sys\nprint(sys.path)''' diff --git a/test/test_inference/test_gradual/test_conversion.py b/test/test_inference/test_gradual/test_conversion.py index 123858d6..c57cb188 100644 --- a/test/test_inference/test_gradual/test_conversion.py +++ b/test/test_inference/test_gradual/test_conversion.py @@ -28,10 +28,10 @@ def test_sqlite3_conversion(Script): def test_conversion_of_stub_only(Script): project = Project(os.path.join(root_dir, 'test', 'completion', 'stub_folder')) code = 'import stub_only; stub_only.in_stub_only' - d1, = Script(code, _project=project).goto() + d1, = Script(code, project=project).goto() assert d1.is_stub() - script = Script(path=d1.module_path, _project=project) + script = Script(path=d1.module_path, project=project) d2, = script.goto(line=d1.line, column=d1.column) assert d2.is_stub() assert d2.module_path == d1.module_path @@ -42,7 +42,7 @@ def test_conversion_of_stub_only(Script): def test_goto_on_file(Script): project = Project(os.path.join(root_dir, 'test', 'completion', 'stub_folder')) - script = Script('import stub_only; stub_only.Foo', _project=project) + script = Script('import stub_only; stub_only.Foo', project=project) d1, = script.goto() v, = d1._name.infer() foo, bar, obj = v.py__mro__() @@ -51,7 +51,7 @@ def test_goto_on_file(Script): assert obj.py__name__() == 'object' # Make sure we go to Bar, because Foo is a bit before: `class Foo(Bar):` - script = Script(path=d1.module_path, _project=project) + script = Script(path=d1.module_path, project=project) d2, = script.goto(line=d1.line, column=d1.column + 4) assert d2.name == 'Bar' diff --git a/test/test_inference/test_gradual/test_stub_loading.py b/test/test_inference/test_gradual/test_stub_loading.py index 7d62a24c..ab64f00c 100644 --- a/test/test_inference/test_gradual/test_stub_loading.py +++ b/test/test_inference/test_gradual/test_stub_loading.py @@ -9,7 +9,7 @@ import pytest def ScriptInStubFolder(Script): path = get_example_dir('stub_packages') project = Project(path, sys_path=[path], smart_sys_path=False) - return partial(Script, _project=project) + return partial(Script, project=project) @pytest.mark.parametrize( diff --git a/test/test_inference/test_gradual/test_stubs.py b/test/test_inference/test_gradual/test_stubs.py index 8d7bdb02..1d82dd0b 100644 --- a/test/test_inference/test_gradual/test_stubs.py +++ b/test/test_inference/test_gradual/test_stubs.py @@ -54,7 +54,7 @@ def test_infer_and_goto(Script, code, full_name, has_stub, has_python, way, has_python = False project = Project(os.path.join(root_dir, 'test', 'completion', 'stub_folder')) - s = Script(code, _project=project) + s = Script(code, project=project) prefer_stubs = kwargs['prefer_stubs'] only_stubs = kwargs['only_stubs'] diff --git a/test/test_inference/test_imports.py b/test/test_inference/test_imports.py index 9ec393ed..f22f3272 100644 --- a/test/test_inference/test_imports.py +++ b/test/test_inference/test_imports.py @@ -363,7 +363,7 @@ def test_relative_imports_with_multiple_similar_directories(Script, path, empty_ script = Script( "from . ", path=os.path.join(dir, path), - _project=project, + project=project, ) name, import_ = script.complete() assert import_.name == 'import' @@ -376,14 +376,14 @@ def test_relative_imports_with_outside_paths(Script): script = Script( "from ...", path=os.path.join(dir, 'api/whatever/test_this.py'), - _project=project, + project=project, ) assert [c.name for c in script.complete()] == ['api', 'whatever'] script = Script( "from " + '.' * 100, path=os.path.join(dir, 'api/whatever/test_this.py'), - _project=project, + project=project, ) assert not script.complete() @@ -391,13 +391,13 @@ def test_relative_imports_with_outside_paths(Script): @cwd_at('test/examples/issue1209/api/whatever/') def test_relative_imports_without_path(Script): project = Project('.', sys_path=[], smart_sys_path=False) - script = Script("from . ", _project=project) + script = Script("from . ", project=project) assert [c.name for c in script.complete()] == ['api_test1', 'import'] - script = Script("from .. ", _project=project) + script = Script("from .. ", project=project) assert [c.name for c in script.complete()] == ['import', 'whatever'] - script = Script("from ... ", _project=project) + script = Script("from ... ", project=project) assert [c.name for c in script.complete()] == ['api', 'import', 'whatever'] From 4a1d9a9116c0cb9c386fca89f8e145a6948e76ae Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 30 Jan 2020 21:02:47 +0100 Subject: [PATCH 02/20] Use project instead of sys_path parameter in tests --- test/test_api/test_classes.py | 5 ++-- test/test_api/test_full_name.py | 5 ++-- test/test_api/test_unicode.py | 4 ++- .../test_implicit_namespace_package.py | 25 +++++++++++-------- test/test_inference/test_imports.py | 24 +++++++++++------- test/test_inference/test_namespace_package.py | 7 +++--- 6 files changed, 42 insertions(+), 28 deletions(-) diff --git a/test/test_api/test_classes.py b/test/test_api/test_classes.py index 4a17c572..a037133b 100644 --- a/test/test_api/test_classes.py +++ b/test/test_api/test_classes.py @@ -129,10 +129,11 @@ def test_completion_docstring(Script, jedi_path): Jedi should follow imports in certain conditions """ def docstr(src, result): - c = Script(src, sys_path=[jedi_path]).complete()[0] + c = Script(src, project=project).complete()[0] assert c.docstring(raw=True, fast=False) == cleandoc(result) - c = Script('import jedi\njed', sys_path=[jedi_path]).complete()[0] + project = jedi.Project('.', sys_path=[jedi_path]) + c = Script('import jedi\njed', project=project).complete()[0] assert c.docstring(fast=False) == cleandoc(jedi_doc) docstr('import jedi\njedi.Scr', cleandoc(jedi.Script.__doc__)) diff --git a/test/test_api/test_full_name.py b/test/test_api/test_full_name.py index 6858b6ca..5944ad72 100644 --- a/test/test_api/test_full_name.py +++ b/test/test_api/test_full_name.py @@ -97,9 +97,10 @@ def test_sub_module(Script, jedi_path): path. """ sys_path = [jedi_path] - defs = Script('from jedi.api import classes; classes', sys_path=sys_path).infer() + project = jedi.Project('.', sys_path=sys_path) + defs = Script('from jedi.api import classes; classes', project=project).infer() assert [d.full_name for d in defs] == ['jedi.api.classes'] - defs = Script('import jedi.api; jedi.api', sys_path=sys_path).infer() + defs = Script('import jedi.api; jedi.api', project=project).infer() assert [d.full_name for d in defs] == ['jedi.api'] diff --git a/test/test_api/test_unicode.py b/test/test_api/test_unicode.py index 015048cd..f7f7ec45 100644 --- a/test/test_api/test_unicode.py +++ b/test/test_api/test_unicode.py @@ -3,6 +3,7 @@ All character set and unicode related tests. """ from jedi._compatibility import u, unicode +from jedi import Project def test_unicode_script(Script): @@ -70,7 +71,8 @@ def test_wrong_encoding(Script, tmpdir): # Use both latin-1 and utf-8 (a really broken file). x.write_binary(u'foobar = 1\nä'.encode('latin-1') + u'ä'.encode('utf-8')) - c, = Script('import x; x.foo', sys_path=[tmpdir.strpath]).complete() + project = Project('.', sys_path=[tmpdir.strpath]) + c, = Script('import x; x.foo', project=project).complete() assert c.name == 'foobar' diff --git a/test/test_inference/test_implicit_namespace_package.py b/test/test_inference/test_implicit_namespace_package.py index 2df9de0c..0b3bee71 100644 --- a/test/test_inference/test_implicit_namespace_package.py +++ b/test/test_inference/test_implicit_namespace_package.py @@ -3,6 +3,7 @@ from os.path import dirname import pytest from test.helpers import get_example_dir, example_dir +from jedi import Project @pytest.fixture(autouse=True) @@ -14,9 +15,10 @@ def skip_not_supported_versions(environment): def test_implicit_namespace_package(Script): sys_path = [get_example_dir('implicit_namespace_package', 'ns1'), get_example_dir('implicit_namespace_package', 'ns2')] + project = Project('.', sys_path=sys_path) def script_with_path(*args, **kwargs): - return Script(sys_path=sys_path, *args, **kwargs) + return Script(project=project, *args, **kwargs) # goto definition assert script_with_path('from pkg import ns1_file').infer() @@ -55,15 +57,14 @@ def test_implicit_namespace_package(Script): def test_implicit_nested_namespace_package(Script): code = 'from implicit_nested_namespaces.namespace.pkg.module import CONST' - sys_path = [example_dir] - - script = Script(sys_path=sys_path, source=code, line=1, column=61) + project = Project('.', sys_path=[example_dir]) + script = Script(project=project, source=code, line=1, column=61) result = script.infer() assert len(result) == 1 - implicit_pkg, = Script(code, sys_path=sys_path).infer(column=10) + implicit_pkg, = Script(code, project=project).infer(column=10) assert implicit_pkg.type == 'module' assert implicit_pkg.module_path is None @@ -71,9 +72,8 @@ def test_implicit_nested_namespace_package(Script): def test_implicit_namespace_package_import_autocomplete(Script): CODE = 'from implicit_name' - sys_path = [example_dir] - - script = Script(sys_path=sys_path, source=CODE) + project = Project('.', sys_path=[example_dir]) + script = Script(project=project, source=CODE) compl = script.complete() assert [c.name for c in compl] == ['implicit_namespace_package'] @@ -83,7 +83,8 @@ def test_namespace_package_in_multiple_directories_autocompletion(Script): sys_path = [get_example_dir('implicit_namespace_package', 'ns1'), get_example_dir('implicit_namespace_package', 'ns2')] - script = Script(sys_path=sys_path, source=CODE) + project = Project('.', sys_path=sys_path) + script = Script(project=project, source=CODE) compl = script.complete() assert set(c.name for c in compl) == set(['ns1_file', 'ns2_file']) @@ -92,7 +93,8 @@ def test_namespace_package_in_multiple_directories_goto_definition(Script): CODE = 'from pkg import ns1_file' sys_path = [get_example_dir('implicit_namespace_package', 'ns1'), get_example_dir('implicit_namespace_package', 'ns2')] - script = Script(sys_path=sys_path, source=CODE) + project = Project('.', sys_path=sys_path) + script = Script(project=project, source=CODE) result = script.infer() assert len(result) == 1 @@ -102,6 +104,7 @@ def test_namespace_name_autocompletion_full_name(Script): sys_path = [get_example_dir('implicit_namespace_package', 'ns1'), get_example_dir('implicit_namespace_package', 'ns2')] - script = Script(sys_path=sys_path, source=CODE) + project = Project('.', sys_path=sys_path) + script = Script(project=project, source=CODE) compl = script.complete() assert set(c.full_name for c in compl) == set(['pkg']) diff --git a/test/test_inference/test_imports.py b/test/test_inference/test_imports.py index f22f3272..5661371f 100644 --- a/test/test_inference/test_imports.py +++ b/test/test_inference/test_imports.py @@ -44,7 +44,9 @@ pkg_zip_path = get_example_dir('zipped_imports', 'pkg.zip') def test_find_module_package_zipped(Script, inference_state, environment): sys_path = environment.get_sys_path() + [pkg_zip_path] - script = Script('import pkg; pkg.mod', sys_path=sys_path) + + project = Project('.', sys_path=sys_path) + script = Script('import pkg; pkg.mod', project=project) assert len(script.complete()) == 1 file_io, is_package = inference_state.compiled_subprocess.get_module_info( @@ -86,7 +88,7 @@ def test_find_module_package_zipped(Script, inference_state, environment): def test_correct_zip_package_behavior(Script, inference_state, environment, code, file, package, path, skip_python2): sys_path = environment.get_sys_path() + [pkg_zip_path] - pkg, = Script(code, sys_path=sys_path).infer() + pkg, = Script(code, project=Project('.', sys_path=sys_path)).infer() value, = pkg._name.infer() assert value.py__file__() == os.path.join(pkg_zip_path, 'pkg', file) assert '.'.join(value.py__package__()) == package @@ -98,7 +100,7 @@ def test_correct_zip_package_behavior(Script, inference_state, environment, code def test_find_module_not_package_zipped(Script, inference_state, environment): path = get_example_dir('zipped_imports', 'not_pkg.zip') sys_path = environment.get_sys_path() + [path] - script = Script('import not_pkg; not_pkg.val', sys_path=sys_path) + script = Script('import not_pkg; not_pkg.val', project=Project('.', sys_path=sys_path)) assert len(script.complete()) == 1 file_io, is_package = inference_state.compiled_subprocess.get_module_info( @@ -144,7 +146,7 @@ def test_flask_ext(Script, code, name): """flask.ext.foo is really imported from flaskext.foo or flask_foo. """ path = get_example_dir('flask-site-packages') - completions = Script(code, sys_path=[path]).complete() + completions = Script(code, project=Project('.', sys_path=[path])).complete() assert name in [c.name for c in completions] @@ -166,10 +168,14 @@ def test_cache_works_with_sys_path_param(Script, tmpdir): bar_path = tmpdir.join('bar') foo_path.join('module.py').write('foo = 123', ensure=True) bar_path.join('module.py').write('bar = 123', ensure=True) - foo_completions = Script('import module; module.', - sys_path=[foo_path.strpath]).complete() - bar_completions = Script('import module; module.', - sys_path=[bar_path.strpath]).complete() + foo_completions = Script( + 'import module; module.', + project=Project('.', sys_path=[foo_path.strpath]), + ).complete() + bar_completions = Script( + 'import module; module.', + project=Project('.', sys_path=[bar_path.strpath]), + ).complete() assert 'foo' in [c.name for c in foo_completions] assert 'bar' not in [c.name for c in foo_completions] @@ -460,7 +466,7 @@ def test_import_needed_modules_by_jedi(Script, environment, tmpdir, name): script = Script( 'import ' + name, path=tmpdir.join('something.py').strpath, - sys_path=[tmpdir.strpath] + environment.get_sys_path(), + project=Project('.', sys_path=[tmpdir.strpath] + environment.get_sys_path()), ) module, = script.infer() assert module._inference_state.builtins_module.py__file__() != module_path diff --git a/test/test_inference/test_namespace_package.py b/test/test_inference/test_namespace_package.py index f6b34151..0867af85 100644 --- a/test/test_inference/test_namespace_package.py +++ b/test/test_inference/test_namespace_package.py @@ -4,6 +4,7 @@ import pytest import py from ..helpers import get_example_dir, example_dir +from jedi import Project SYS_PATH = [get_example_dir('namespace_package', 'ns1'), @@ -11,7 +12,7 @@ SYS_PATH = [get_example_dir('namespace_package', 'ns1'), def script_with_path(Script, *args, **kwargs): - return Script(sys_path=SYS_PATH, *args, **kwargs) + return Script(project=Project('.', sys_path=SYS_PATH), *args, **kwargs) def test_goto_definition(Script): @@ -69,8 +70,8 @@ def test_nested_namespace_package(Script): code = 'from nested_namespaces.namespace.pkg import CONST' sys_path = [example_dir] - - script = Script(sys_path=sys_path, source=code) + project = Project('.', sys_path=sys_path) + script = Script(project=project, source=code) result = script.infer(line=1, column=45) From 251ff447bc503973ef287753bf06373e34978c49 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 Jan 2020 00:08:24 +0100 Subject: [PATCH 03/20] Add added_sys_path to Project, fixes #1334 --- jedi/api/project.py | 8 ++++++-- test/test_api/test_project.py | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/jedi/api/project.py b/jedi/api/project.py index 6b76c8dc..ac70b26a 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -61,11 +61,13 @@ class Project(object): :param sys_path: list of str. You can override the sys path if you want. By default the ``sys.path.`` is generated from the environment (virtualenvs, etc). + :param added_sys_path: list of str. Adds these paths at the end of the + sys path. :param smart_sys_path: If this is enabled (default), adds paths from local directories. Otherwise you will have to rely on your packages being properly configured on the ``sys.path``. """ - def py2_comp(path, environment=None, sys_path=None, + def py2_comp(path, environment=None, sys_path=None, added_sys_path=True, smart_sys_path=True, _django=False): self._path = os.path.abspath(path) if isinstance(environment, SameEnvironment): @@ -74,6 +76,8 @@ class Project(object): self._sys_path = sys_path self._smart_sys_path = smart_sys_path self._django = _django + self.added_sys_path = [] + """The sys path that is going to be added at the end of the """ py2_comp(path, **kwargs) @@ -91,7 +95,7 @@ class Project(object): sys_path.remove('') except ValueError: pass - return sys_path + return sys_path + self.added_sys_path @inference_state_as_method_param_cache() def _get_sys_path(self, inference_state, environment=None, diff --git a/test/test_api/test_project.py b/test/test_api/test_project.py index f8bc8c60..7eeced4f 100644 --- a/test/test_api/test_project.py +++ b/test/test_api/test_project.py @@ -2,6 +2,7 @@ import os from ..helpers import get_example_dir, set_cwd, root_dir from jedi import Interpreter +from jedi.api import Project, get_default_project def test_django_default_project(Script): @@ -22,3 +23,10 @@ def test_interpreter_project_path(): with set_cwd(dir): project = Interpreter('', [locals()])._inference_state.project assert project._path == dir + + +def test_added_sys_path(inference_state): + project = get_default_project() + p = '/some_random_path' + project.added_sys_path = [p] + assert p in project._get_base_sys_path(inference_state) From d02af44331a97e148f987695d4a6a613bbf9e171 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 Jan 2020 00:21:46 +0100 Subject: [PATCH 04/20] Make it possible to use get_default_project directly from Jedi --- jedi/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/__init__.py b/jedi/__init__.py index e1356feb..5ee509e8 100644 --- a/jedi/__init__.py +++ b/jedi/__init__.py @@ -41,7 +41,7 @@ from jedi import settings from jedi.api.environment import find_virtualenvs, find_system_environments, \ get_default_environment, InvalidPythonEnvironment, create_environment, \ get_system_environment -from jedi.api.project import Project +from jedi.api.project import Project, get_default_project from jedi.api.exceptions import InternalError # Finally load the internal plugins. This is only internal. from jedi.plugins import registry From e5ec2a3adfe6d5fcae173167c6030b66001d3b19 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 Jan 2020 01:46:55 +0100 Subject: [PATCH 05/20] Introduce two new Project params: python_path, python_version --- jedi/api/project.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/jedi/api/project.py b/jedi/api/project.py index ac70b26a..0f5e84a7 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -3,7 +3,7 @@ import json from jedi._compatibility import FileNotFoundError, PermissionError, IsADirectoryError from jedi.api.environment import SameEnvironment, \ - get_cached_default_environment + get_cached_default_environment, get_system_environment, create_environment from jedi.api.exceptions import WrongVersion from jedi._compatibility import force_unicode from jedi.inference.sys_path import discover_buildout_paths @@ -30,8 +30,6 @@ def _force_unicode_list(lst): class Project(object): - # TODO serialize environment - _serializer_ignore_attributes = ('_environment',) _environment = None @staticmethod @@ -58,6 +56,10 @@ class Project(object): def __init__(self, path, **kwargs): """ :param path: The base path for this project. + :param python_path: The Python executable path, typically the path of a + virtual environment. + :param python_version: The version string of the Python environment to + be loaded, e.g. `"3"` or `"3.8"`. :param sys_path: list of str. You can override the sys path if you want. By default the ``sys.path.`` is generated from the environment (virtualenvs, etc). @@ -67,12 +69,14 @@ class Project(object): local directories. Otherwise you will have to rely on your packages being properly configured on the ``sys.path``. """ - def py2_comp(path, environment=None, sys_path=None, added_sys_path=True, - smart_sys_path=True, _django=False): + def py2_comp(path, python_path=None, python_version=None, sys_path=None, + added_sys_path=True, smart_sys_path=True, _django=False): + if python_version is not None and python_path is not None: + raise ValueError('You cannot use both python_version and python_path') self._path = os.path.abspath(path) - if isinstance(environment, SameEnvironment): - self._environment = environment + self._python_path = python_path + self._python_version = python_version self._sys_path = sys_path self._smart_sys_path = smart_sys_path self._django = _django @@ -140,16 +144,20 @@ class Project(object): def save(self): data = dict(self.__dict__) - for attribute in self._serializer_ignore_attributes: - data.pop(attribute, None) + data.pop('_environment', None) + data = {k.lstrip('_'): v for k, v in data.items()} with open(self._get_json_path(self._path), 'wb') as f: return json.dump((_SERIALIZER_VERSION, data), f) def get_environment(self): if self._environment is None: - return get_cached_default_environment() - + if self._python_path is not None: + self._environment = create_environment(self._python_path, safe=False) + elif self._python_version is not None: + self._environment = get_system_environment(self._python_version) + else: + self._environment = get_cached_default_environment() return self._environment def __repr__(self): From d09882f9701aa44bc22583bd890f41a2e690742e Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 Jan 2020 01:50:52 +0100 Subject: [PATCH 06/20] Remove django from the project API --- jedi/api/project.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/jedi/api/project.py b/jedi/api/project.py index 0f5e84a7..1b0b1a9f 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -70,7 +70,7 @@ class Project(object): being properly configured on the ``sys.path``. """ def py2_comp(path, python_path=None, python_version=None, sys_path=None, - added_sys_path=True, smart_sys_path=True, _django=False): + added_sys_path=True, smart_sys_path=True): if python_version is not None and python_path is not None: raise ValueError('You cannot use both python_version and python_path') self._path = os.path.abspath(path) @@ -79,7 +79,7 @@ class Project(object): self._python_version = python_version self._sys_path = sys_path self._smart_sys_path = smart_sys_path - self._django = _django + self._django = False self.added_sys_path = [] """The sys path that is going to be added at the end of the """ @@ -204,7 +204,9 @@ def get_default_project(path=None): first_no_init_file = dir if _is_django_path(dir): - return Project(dir, _django=True) + project = Project(dir) + project._django = True + return project if probable_path is None and _is_potential_project(dir): probable_path = dir From a05628443e95409c7e7ced1063c1c25c2bdabd72 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 Jan 2020 02:14:34 +0100 Subject: [PATCH 07/20] Make sure serialization works for projects --- jedi/api/project.py | 24 +++++++++++++++++------- test/test_api/test_project.py | 8 ++++++++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/jedi/api/project.py b/jedi/api/project.py index 1b0b1a9f..e46d136a 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -1,4 +1,5 @@ import os +import errno import json from jedi._compatibility import FileNotFoundError, PermissionError, IsADirectoryError @@ -32,9 +33,13 @@ def _force_unicode_list(lst): class Project(object): _environment = None + @staticmethod + def _get_config_folder_path(base_path): + return os.path.join(base_path, _CONFIG_FOLDER) + @staticmethod def _get_json_path(base_path): - return os.path.join(base_path, _CONFIG_FOLDER, 'project.json') + return os.path.join(Project._get_config_folder_path(base_path), 'project.json') @classmethod def load(cls, path): @@ -45,9 +50,7 @@ class Project(object): version, data = json.load(f) if version == 1: - self = cls.__new__() - self.__dict__.update(data) - return self + return cls(**data) else: raise WrongVersion( "The Jedi version of this project seems newer than what we can handle." @@ -70,7 +73,7 @@ class Project(object): being properly configured on the ``sys.path``. """ def py2_comp(path, python_path=None, python_version=None, sys_path=None, - added_sys_path=True, smart_sys_path=True): + added_sys_path=(), smart_sys_path=True): if python_version is not None and python_path is not None: raise ValueError('You cannot use both python_version and python_path') self._path = os.path.abspath(path) @@ -80,7 +83,7 @@ class Project(object): self._sys_path = sys_path self._smart_sys_path = smart_sys_path self._django = False - self.added_sys_path = [] + self.added_sys_path = list(added_sys_path) """The sys path that is going to be added at the end of the """ py2_comp(path, **kwargs) @@ -145,9 +148,16 @@ class Project(object): def save(self): data = dict(self.__dict__) data.pop('_environment', None) + data.pop('_django', None) # TODO make django setting public? data = {k.lstrip('_'): v for k, v in data.items()} - with open(self._get_json_path(self._path), 'wb') as f: + # TODO when dropping Python 2 use pathlib.Path.mkdir(parents=True, exist_ok=True) + try: + os.makedirs(self._get_config_folder_path(self._path)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + with open(self._get_json_path(self._path), 'w') as f: return json.dump((_SERIALIZER_VERSION, data), f) def get_environment(self): diff --git a/test/test_api/test_project.py b/test/test_api/test_project.py index 7eeced4f..b5ef5db0 100644 --- a/test/test_api/test_project.py +++ b/test/test_api/test_project.py @@ -30,3 +30,11 @@ def test_added_sys_path(inference_state): p = '/some_random_path' project.added_sys_path = [p] assert p in project._get_base_sys_path(inference_state) + + +def test_load_save_project(tmpdir): + project = Project(tmpdir.strpath, added_sys_path=['/foo']) + project.save() + + loaded = Project.load(tmpdir.strpath) + assert loaded.added_sys_path == ['/foo'] From e7a77e438dd83a444b48285c10bcecafc6666487 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 Jan 2020 02:15:24 +0100 Subject: [PATCH 08/20] Remove python_version again, it might not be needed --- jedi/api/project.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/jedi/api/project.py b/jedi/api/project.py index e46d136a..8b9c9228 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -3,8 +3,7 @@ import errno import json from jedi._compatibility import FileNotFoundError, PermissionError, IsADirectoryError -from jedi.api.environment import SameEnvironment, \ - get_cached_default_environment, get_system_environment, create_environment +from jedi.api.environment import get_cached_default_environment, create_environment from jedi.api.exceptions import WrongVersion from jedi._compatibility import force_unicode from jedi.inference.sys_path import discover_buildout_paths @@ -61,8 +60,6 @@ class Project(object): :param path: The base path for this project. :param python_path: The Python executable path, typically the path of a virtual environment. - :param python_version: The version string of the Python environment to - be loaded, e.g. `"3"` or `"3.8"`. :param sys_path: list of str. You can override the sys path if you want. By default the ``sys.path.`` is generated from the environment (virtualenvs, etc). @@ -72,14 +69,11 @@ class Project(object): local directories. Otherwise you will have to rely on your packages being properly configured on the ``sys.path``. """ - def py2_comp(path, python_path=None, python_version=None, sys_path=None, + def py2_comp(path, python_path=None, sys_path=None, added_sys_path=(), smart_sys_path=True): - if python_version is not None and python_path is not None: - raise ValueError('You cannot use both python_version and python_path') self._path = os.path.abspath(path) self._python_path = python_path - self._python_version = python_version self._sys_path = sys_path self._smart_sys_path = smart_sys_path self._django = False @@ -164,8 +158,6 @@ class Project(object): if self._environment is None: if self._python_path is not None: self._environment = create_environment(self._python_path, safe=False) - elif self._python_version is not None: - self._environment = get_system_environment(self._python_version) else: self._environment = get_cached_default_environment() return self._environment From 8ff2ea4b38e4e2677d44936bdff1856efac979e2 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 31 Jan 2020 13:09:28 +0100 Subject: [PATCH 09/20] Make sure to not load unsafe modules anymore if they are not on the sys path, fixes #760 --- jedi/api/project.py | 30 +++++++++++++++------------ jedi/inference/__init__.py | 2 +- jedi/inference/imports.py | 4 ++++ test/test_api/test_project.py | 2 +- test/test_inference/test_extension.py | 28 ++++++++++++++++++++----- test/test_inference/test_pyc.py | 13 +++++++++--- 6 files changed, 56 insertions(+), 23 deletions(-) diff --git a/jedi/api/project.py b/jedi/api/project.py index 8b9c9228..9b418e2a 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -60,6 +60,11 @@ class Project(object): :param path: The base path for this project. :param python_path: The Python executable path, typically the path of a virtual environment. + :param load_unsafe_extensions: Loads extensions that are not in the + sys path and in the local directories. With this option enabled, + this is potentially unsafe if you clone a git repository and + analyze it's code, because those compiled extensions will be + important and therefore have execution privileges. :param sys_path: list of str. You can override the sys path if you want. By default the ``sys.path.`` is generated from the environment (virtualenvs, etc). @@ -69,13 +74,14 @@ class Project(object): local directories. Otherwise you will have to rely on your packages being properly configured on the ``sys.path``. """ - def py2_comp(path, python_path=None, sys_path=None, - added_sys_path=(), smart_sys_path=True): + def py2_comp(path, python_path=None, load_unsafe_extensions=False, + sys_path=None, added_sys_path=(), smart_sys_path=True): self._path = os.path.abspath(path) self._python_path = python_path self._sys_path = sys_path self._smart_sys_path = smart_sys_path + self._load_unsafe_extensions = load_unsafe_extensions self._django = False self.added_sys_path = list(added_sys_path) """The sys path that is going to be added at the end of the """ @@ -83,20 +89,14 @@ class Project(object): py2_comp(path, **kwargs) @inference_state_as_method_param_cache() - def _get_base_sys_path(self, inference_state, environment=None): - if self._sys_path is not None: - return self._sys_path - + def _get_base_sys_path(self, inference_state): # The sys path has not been set explicitly. - if environment is None: - environment = self.get_environment() - - sys_path = list(environment.get_sys_path()) + sys_path = list(self.get_environment().get_sys_path()) try: sys_path.remove('') except ValueError: pass - return sys_path + self.added_sys_path + return sys_path @inference_state_as_method_param_cache() def _get_sys_path(self, inference_state, environment=None, @@ -105,10 +105,14 @@ class Project(object): Keep this method private for all users of jedi. However internally this one is used like a public method. """ - suffixed = [] + suffixed = list(self.added_sys_path) prefixed = [] - sys_path = list(self._get_base_sys_path(inference_state, environment)) + if self._sys_path is None: + sys_path = list(self._get_base_sys_path(inference_state)) + else: + sys_path = list(self._sys_path) + if self._smart_sys_path: prefixed.append(self._path) diff --git a/jedi/inference/__init__.py b/jedi/inference/__init__.py index 7606be42..91f05841 100644 --- a/jedi/inference/__init__.py +++ b/jedi/inference/__init__.py @@ -142,7 +142,7 @@ class InferenceState(object): def get_sys_path(self, **kwargs): """Convenience function""" - return self.project._get_sys_path(self, environment=self.environment, **kwargs) + return self.project._get_sys_path(self, **kwargs) def infer(self, context, name): def_ = name.get_definition(import_name_always=True) diff --git a/jedi/inference/imports.py b/jedi/inference/imports.py index bbf944ed..bc1ce0b9 100644 --- a/jedi/inference/imports.py +++ b/jedi/inference/imports.py @@ -454,8 +454,12 @@ def _load_python_module(inference_state, file_io, def _load_builtin_module(inference_state, import_names=None, sys_path=None): + project = inference_state.project if sys_path is None: sys_path = inference_state.get_sys_path() + if not project._load_unsafe_extensions: + safe_paths = project._get_base_sys_path(inference_state) + sys_path = [p for p in sys_path if p in safe_paths] dotted_name = '.'.join(import_names) assert dotted_name is not None diff --git a/test/test_api/test_project.py b/test/test_api/test_project.py index b5ef5db0..c3d016f3 100644 --- a/test/test_api/test_project.py +++ b/test/test_api/test_project.py @@ -29,7 +29,7 @@ def test_added_sys_path(inference_state): project = get_default_project() p = '/some_random_path' project.added_sys_path = [p] - assert p in project._get_base_sys_path(inference_state) + assert p in project._get_sys_path(inference_state) def test_load_save_project(tmpdir): diff --git a/test/test_inference/test_extension.py b/test/test_inference/test_extension.py index 6e6f5899..e488f4fd 100644 --- a/test/test_inference/test_extension.py +++ b/test/test_inference/test_extension.py @@ -35,8 +35,9 @@ def test_get_signatures_stdlib(Script): # Check only on linux 64 bit platform and Python3.4. @pytest.mark.skipif('sys.platform != "linux" or sys.maxsize <= 2**32 or sys.version_info[:2] != (3, 4)') +@pytest.mark.parametrize('load_unsafe_extensions', [False, True]) @cwd_at('test/examples') -def test_init_extension_module(Script): +def test_init_extension_module(Script, load_unsafe_extensions): """ ``__init__`` extension modules are also packages and Jedi should understand that. @@ -50,8 +51,25 @@ def test_init_extension_module(Script): This is also why this test only runs on certain systems (and Python 3.4). """ - s = jedi.Script('import init_extension_module as i\ni.', path='not_existing.py') - assert 'foo' in [c.name for c in s.complete()] + project = jedi.Project('.', load_unsafe_extensions=load_unsafe_extensions) + s = jedi.Script( + 'import init_extension_module as i\ni.', + path='not_existing.py', + project=project, + ) + if load_unsafe_extensions: + assert 'foo' in [c.name for c in s.complete()] + else: + assert 'foo' not in [c.name for c in s.complete()] - s = jedi.Script('from init_extension_module import foo\nfoo', path='not_existing.py') - assert ['foo'] == [c.name for c in s.complete()] + s = jedi.Script( + 'from init_extension_module import foo\nfoo', + path='not_existing.py', + project=project, + ) + c, = s.complete() + assert c.name == 'foo' + if load_unsafe_extensions: + assert c.infer() + else: + assert not c.infer() diff --git a/test/test_inference/test_pyc.py b/test/test_inference/test_pyc.py index 615fd1cd..35d51327 100644 --- a/test/test_inference/test_pyc.py +++ b/test/test_inference/test_pyc.py @@ -55,7 +55,8 @@ def pyc_project_path(tmpdir): shutil.rmtree(path) -def test_pyc(pyc_project_path, environment): +@pytest.mark.parametrize('load_unsafe_extensions', [False, True]) +def test_pyc(pyc_project_path, environment, load_unsafe_extensions): """ The list of completion must be greater than 2. """ @@ -66,8 +67,14 @@ def test_pyc(pyc_project_path, environment): # we also have the same version and it's easier to debug. environment = SameEnvironment() environment = environment + project = jedi.Project(pyc_project_path, load_unsafe_extensions=load_unsafe_extensions) s = jedi.Script( "from dummy_package import dummy; dummy.", path=path, - environment=environment) - assert len(s.complete()) >= 2 + environment=environment, + project=project, + ) + if load_unsafe_extensions: + assert len(s.complete()) >= 2 + else: + assert not s.complete() From 2c62166ff6607787275f6d8f691c748b47e0a841 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 3 Feb 2020 22:06:12 +0100 Subject: [PATCH 10/20] Get parser errors working, fixes #1488 --- jedi/api/__init__.py | 4 +++ jedi/api/errors.py | 36 +++++++++++++++++++ test/test_api/test_syntax_errors.py | 54 +++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+) create mode 100644 jedi/api/errors.py create mode 100644 test/test_api/test_syntax_errors.py diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index 39f1c904..d7b2d5ae 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -30,6 +30,7 @@ from jedi.api.completion import Completion from jedi.api.keywords import KeywordName from jedi.api.environment import InterpreterEnvironment from jedi.api.project import get_default_project, Project +from jedi.api.errors import parso_to_jedi_errors from jedi.inference import InferenceState from jedi.inference import imports from jedi.inference.references import find_references @@ -504,6 +505,9 @@ class Script(object): """ return self._names(**kwargs) # Python 2... + def get_syntax_errors(self): + return parso_to_jedi_errors(self._grammar, self._module_node) + def _names(self, all_scopes=False, definitions=True, references=False): def def_ref_filter(_def): is_def = _def._name.tree_name.is_definition() diff --git a/jedi/api/errors.py b/jedi/api/errors.py new file mode 100644 index 00000000..e86f9212 --- /dev/null +++ b/jedi/api/errors.py @@ -0,0 +1,36 @@ +""" +This file is about errors in Python files and not about exception handling in +Jedi. +""" + + +def parso_to_jedi_errors(grammar, module_node): + return [SyntaxError(e) for e in grammar.iter_errors(module_node)] + + +class SyntaxError(object): + def __init__(self, parso_error): + self._parso_error = parso_error + + @property + def line(self): + return self._parso_error.start_pos[0] + + @property + def column(self): + return self._parso_error.start_pos[1] + + @property + def until_line(self): + return self._parso_error.end_pos[0] + + @property + def until_column(self): + return self._parso_error.end_pos[1] + + def __repr__(self): + return '<%s from=%s to=%s>' % ( + self.__class__.__name__, + self._parso_error.start_pos, + self._parso_error.end_pos, + ) diff --git a/test/test_api/test_syntax_errors.py b/test/test_api/test_syntax_errors.py new file mode 100644 index 00000000..01e99688 --- /dev/null +++ b/test/test_api/test_syntax_errors.py @@ -0,0 +1,54 @@ +""" +These tests test Jedi's Parso usage. Basically there's not a lot of tests here, +because we're just checking if the API works. Bugfixes should be done in parso, +mostly. +""" + +from textwrap import dedent + +import pytest + + +@pytest.mark.parametrize( + 'code, line, column, until_line, until_column', [ + ('?\n', 1, 0, 1, 1), + ('x %% y', 1, 3, 1, 4), + ('"""\n\n', 1, 0, 3, 0), + ('(1, 2\n', 2, 0, 2, 0), + ('foo(1, 2\ndef x(): pass', 2, 0, 2, 3), + ] +) +def test_simple_syntax_errors(Script, code, line, column, until_line, until_column): + e, = Script(code).get_syntax_errors() + assert e.line == line + assert e.column == column + assert e.until_line == until_line + assert e.until_column == until_column + + +@pytest.mark.parametrize( + 'code', [ + 'x % y', + 'def x(x): pass', + 'def x(x):\n pass', + ] +) +def test_no_syntax_errors(Script, code): + assert not Script(code).get_syntax_errors() + + +def test_multi_syntax_error(Script): + code = dedent('''\ + def x(): + 1 + def y() + 1 + 1 + 1 *** 3 + ''') + x, y, power = Script(code).get_syntax_errors() + assert x.line == 2 + assert x.column == 0 + assert y.line == 3 + assert y.column == 7 + assert power.line == 5 + assert power.column == 4 From eb88c483fb75e3e79d92132bd7d4384d8a423479 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 3 Feb 2020 22:27:22 +0100 Subject: [PATCH 11/20] Catch an error with illegal class instances, fixes #1491 --- jedi/inference/compiled/access.py | 4 +++- test/test_api/test_interpreter.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/jedi/inference/compiled/access.py b/jedi/inference/compiled/access.py index f47ae773..f76a645f 100644 --- a/jedi/inference/compiled/access.py +++ b/jedi/inference/compiled/access.py @@ -570,4 +570,6 @@ def _is_class_instance(obj): except AttributeError: return False else: - return cls != type and not issubclass(cls, NOT_CLASS_TYPES) + # The isinstance check for cls is just there so issubclass doesn't + # raise an exception. + return cls != type and isinstance(cls, type) and not issubclass(cls, NOT_CLASS_TYPES) diff --git a/test/test_api/test_interpreter.py b/test/test_api/test_interpreter.py index 160639eb..1ec0b741 100644 --- a/test/test_api/test_interpreter.py +++ b/test/test_api/test_interpreter.py @@ -530,6 +530,16 @@ def test__wrapped__(): assert c.line == syslogs_to_df.__wrapped__.__code__.co_firstlineno + 1 +@pytest.mark.skipif(sys.version_info[0] == 2, reason="Ignore Python 2, because EOL") +def test_illegal_class_instance(): + class X: + __class__ = 1 + X.__name__ = 'asdf' + d, = jedi.Interpreter('foo', [{'foo': X()}]).infer() + v, = d._name.infer() + assert not v.is_instance() + + @pytest.mark.skipif(sys.version_info[0] == 2, reason="Ignore Python 2, because EOL") @pytest.mark.parametrize('module_name', ['sys', 'time', 'unittest.mock']) def test_core_module_completes(module_name): From 66e28eb52ee9fab5609cd57956424551a93dc5e5 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 4 Feb 2020 10:03:55 +0100 Subject: [PATCH 12/20] Move test_api/test_defined_names.py -> test_api/test_names.py --- test/test_api/{test_defined_names.py => test_names.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename test/test_api/{test_defined_names.py => test_names.py} (100%) diff --git a/test/test_api/test_defined_names.py b/test/test_api/test_names.py similarity index 100% rename from test/test_api/test_defined_names.py rename to test/test_api/test_names.py From 692bf5cfb749a652d71c2018e98581f78d271a71 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 4 Feb 2020 10:12:13 +0100 Subject: [PATCH 13/20] Properly identify side effects, fixes #1411 --- jedi/api/classes.py | 6 ++++++ test/test_api/test_names.py | 19 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index b7aaadd3..0456e7c3 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -523,6 +523,12 @@ class BaseDefinition(object): """ return self._name.infer().get_type_hint() + def is_side_effect(self): + tree_name = self._name.tree_name + if tree_name is None: + return False + return tree_name.parent.type == 'trailer' + class Completion(BaseDefinition): """ diff --git a/test/test_api/test_names.py b/test/test_api/test_names.py index 8b1c731e..3d1ed07c 100644 --- a/test/test_api/test_names.py +++ b/test/test_api/test_names.py @@ -4,6 +4,8 @@ Tests for `api.names`. from textwrap import dedent +import pytest + def _assert_definition_names(definitions, names): assert [d.name for d in definitions] == names @@ -167,3 +169,20 @@ def test_no_error(get_names): assert b.name == 'b' assert a20.name == 'a' assert a20.goto() == [a20] + + +@pytest.mark.parametrize( + 'code, index, is_side_effect', [ + ('x', 0, False), + ('x.x', 0, False), + ('x.x', 1, True), + ('def x(x): x.x', 1, False), + ('def x(x): x.x', 3, True), + ('import sys; sys.path', 0, False), + ('import sys; sys.path', 1, False), + ('import sys; sys.path', 2, True), + ] +) +def test_is_side_effect(get_names, code, index, is_side_effect): + names = get_names(code, references=True, all_scopes=True) + assert names[index].is_side_effect() == is_side_effect From 6313934d9498c3546eb09550660391b8eacafdb6 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 4 Feb 2020 19:39:13 +0100 Subject: [PATCH 14/20] Add a docstring for is_side_effect --- jedi/api/classes.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 0456e7c3..37a403af 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -524,6 +524,10 @@ class BaseDefinition(object): return self._name.infer().get_type_hint() def is_side_effect(self): + """ + Checks if a name is defined as ``self.foo = 3``. In case of self, this + function would return False, for foo it would return True. + """ tree_name = self._name.tree_name if tree_name is None: return False From 670d6e86395d99e40feb79bb3ff3722200470884 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 4 Feb 2020 20:12:24 +0100 Subject: [PATCH 15/20] Move is_side_effect to Definition and correct bugs --- jedi/api/classes.py | 20 ++++++++++---------- test/test_api/test_names.py | 11 +++++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 37a403af..14cf2976 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -523,16 +523,6 @@ class BaseDefinition(object): """ return self._name.infer().get_type_hint() - def is_side_effect(self): - """ - Checks if a name is defined as ``self.foo = 3``. In case of self, this - function would return False, for foo it would return True. - """ - tree_name = self._name.tree_name - if tree_name is None: - return False - return tree_name.parent.type == 'trailer' - class Completion(BaseDefinition): """ @@ -717,6 +707,16 @@ class Definition(BaseDefinition): else: return self._name.tree_name.is_definition() + def is_side_effect(self): + """ + Checks if a name is defined as ``self.foo = 3``. In case of self, this + function would return False, for foo it would return True. + """ + tree_name = self._name.tree_name + if tree_name is None: + return False + return tree_name.is_definition() and tree_name.parent.type == 'trailer' + def __eq__(self, other): return self._name.start_pos == other._name.start_pos \ and self.module_path == other.module_path \ diff --git a/test/test_api/test_names.py b/test/test_api/test_names.py index 3d1ed07c..d43a29bb 100644 --- a/test/test_api/test_names.py +++ b/test/test_api/test_names.py @@ -175,12 +175,15 @@ def test_no_error(get_names): 'code, index, is_side_effect', [ ('x', 0, False), ('x.x', 0, False), - ('x.x', 1, True), - ('def x(x): x.x', 1, False), - ('def x(x): x.x', 3, True), + ('x.x', 1, False), + ('x.x = 3', 0, False), + ('x.x = 3', 1, True), + ('def x(x): x.x = 3', 1, False), + ('def x(x): x.x = 3', 3, True), ('import sys; sys.path', 0, False), ('import sys; sys.path', 1, False), - ('import sys; sys.path', 2, True), + ('import sys; sys.path', 2, False), + ('import sys; sys.path = []', 2, True), ] ) def test_is_side_effect(get_names, code, index, is_side_effect): From f2722952e750ff69cd2108f91d4aad02a25c0edb Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 5 Feb 2020 10:01:21 +0100 Subject: [PATCH 16/20] Fix load_unsafe_extensions issue --- test/test_inference/test_extension.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test_inference/test_extension.py b/test/test_inference/test_extension.py index e488f4fd..a63af388 100644 --- a/test/test_inference/test_extension.py +++ b/test/test_inference/test_extension.py @@ -4,7 +4,7 @@ Test compiled module import os import jedi -from ..helpers import cwd_at +from ..helpers import get_example_dir import pytest @@ -34,9 +34,8 @@ def test_get_signatures_stdlib(Script): # Check only on linux 64 bit platform and Python3.4. -@pytest.mark.skipif('sys.platform != "linux" or sys.maxsize <= 2**32 or sys.version_info[:2] != (3, 4)') @pytest.mark.parametrize('load_unsafe_extensions', [False, True]) -@cwd_at('test/examples') +@pytest.mark.skipif('sys.platform != "linux" or sys.maxsize <= 2**32 or sys.version_info[:2] != (3, 4)') def test_init_extension_module(Script, load_unsafe_extensions): """ ``__init__`` extension modules are also packages and Jedi should understand @@ -51,7 +50,8 @@ def test_init_extension_module(Script, load_unsafe_extensions): This is also why this test only runs on certain systems (and Python 3.4). """ - project = jedi.Project('.', load_unsafe_extensions=load_unsafe_extensions) + + project = jedi.Project(get_example_dir(), load_unsafe_extensions=load_unsafe_extensions) s = jedi.Script( 'import init_extension_module as i\ni.', path='not_existing.py', From 14ac0512a91848838e6881a263d72f22869b9837 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 6 Feb 2020 01:47:34 +0100 Subject: [PATCH 17/20] Get rid of cwd modifications in tests --- conftest.py | 7 +++ jedi/inference/imports.py | 5 +- test/helpers.py | 2 + .../test_api_classes_follow_definition.py | 6 ++- test/test_api/test_settings.py | 2 - test/test_inference/test_absolute_import.py | 5 +- .../test_inference/test_buildout_detection.py | 17 +++---- test/test_inference/test_imports.py | 49 ++++++++++--------- test/test_speed.py | 3 +- test/test_utils.py | 5 +- 10 files changed, 55 insertions(+), 46 deletions(-) diff --git a/conftest.py b/conftest.py index 81d0fe54..91e772b3 100644 --- a/conftest.py +++ b/conftest.py @@ -9,6 +9,7 @@ import pytest import jedi from jedi.api.environment import get_system_environment, InterpreterEnvironment from jedi._compatibility import py_version +from test.helpers import test_dir collect_ignore = [ 'setup.py', @@ -109,6 +110,12 @@ def Script(environment): return partial(jedi.Script, environment=environment) +@pytest.fixture(scope='session') +def ScriptWithProject(Script): + project = jedi.Project(test_dir) + return partial(jedi.Script, project=project) + + @pytest.fixture(scope='session') def get_names(Script): return lambda code, **kwargs: Script(code).get_names(**kwargs) diff --git a/jedi/inference/imports.py b/jedi/inference/imports.py index bc1ce0b9..a43e19aa 100644 --- a/jedi/inference/imports.py +++ b/jedi/inference/imports.py @@ -191,18 +191,19 @@ class Importer(object): import_path = base + tuple(import_path) else: path = module_context.py__file__() + project_path = self._inference_state.project._path import_path = list(import_path) if path is None: # If no path is defined, our best guess is that the current # file is edited by a user on the current working # directory. We need to add an initial path, because it # will get removed as the name of the current file. - directory = os.getcwd() + directory = project_path else: directory = os.path.dirname(path) base_import_path, base_directory = _level_to_base_import_path( - self._inference_state.project._path, directory, level, + project_path, directory, level, ) if base_directory is None: # Everything is lost, the relative import does point diff --git a/test/helpers.py b/test/helpers.py index 7e991915..f2782829 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -16,8 +16,10 @@ import os import pytest from os.path import abspath, dirname, join from functools import partial, wraps +from jedi import Project test_dir = dirname(abspath(__file__)) +test_dir_project = Project(test_dir) root_dir = dirname(test_dir) example_dir = join(test_dir, 'examples') diff --git a/test/test_api/test_api_classes_follow_definition.py b/test/test_api/test_api_classes_follow_definition.py index 83ed1c95..4bf3f254 100644 --- a/test/test_api/test_api_classes_follow_definition.py +++ b/test/test_api/test_api_classes_follow_definition.py @@ -1,7 +1,9 @@ +from os.path import join from itertools import chain +from functools import partial import jedi -from ..helpers import cwd_at +from ..helpers import test_dir def test_import_empty(Script): @@ -47,8 +49,8 @@ def test_follow_import_incomplete(Script, environment): assert alias == ['module'] -@cwd_at('test/completion/import_tree') def test_follow_definition_nested_import(Script): + Script = partial(Script, project=jedi.Project(join(test_dir, 'completion', 'import_tree'))) types = check_follow_definition_types(Script, "import pkg.mod1; pkg") assert types == ['module'] diff --git a/test/test_api/test_settings.py b/test/test_api/test_settings.py index 7a9d0c2c..3a21fc76 100644 --- a/test/test_api/test_settings.py +++ b/test/test_api/test_settings.py @@ -4,11 +4,9 @@ import pytest from jedi import api from jedi.inference import imports -from ..helpers import cwd_at @pytest.mark.skipif('True', reason='Skip for now, test case is not really supported.') -@cwd_at('jedi') def test_add_dynamic_mods(Script): fname = '__main__.py' api.settings.additional_dynamic_modules = [fname] diff --git a/test/test_inference/test_absolute_import.py b/test/test_inference/test_absolute_import.py index 3bf9dd2e..72017dc9 100644 --- a/test/test_inference/test_absolute_import.py +++ b/test/test_inference/test_absolute_import.py @@ -2,10 +2,11 @@ Tests ``from __future__ import absolute_import`` (only important for Python 2.X) """ +from jedi import Project from .. import helpers -@helpers.cwd_at("test/examples/absolute_import") def test_can_complete_when_shadowing(Script): - script = Script(path="unittest.py") + path = helpers.get_example_dir('absolute_import', 'unittest.py') + script = Script(path=path, project=Project(helpers.get_example_dir('absolute_import'))) assert script.complete() diff --git a/test/test_inference/test_buildout_detection.py b/test/test_inference/test_buildout_detection.py index 6815aa9b..aa70eca0 100644 --- a/test/test_inference/test_buildout_detection.py +++ b/test/test_inference/test_buildout_detection.py @@ -5,7 +5,7 @@ from jedi._compatibility import force_unicode from jedi.inference.sys_path import _get_parent_dir_with_file, \ _get_buildout_script_paths, check_sys_path_modifications -from ..helpers import cwd_at +from ..helpers import get_example_dir def check_module_test(Script, code): @@ -13,20 +13,18 @@ def check_module_test(Script, code): return check_sys_path_modifications(module_context) -@cwd_at('test/examples/buildout_project/src/proj_name') def test_parent_dir_with_file(Script): - parent = _get_parent_dir_with_file( - os.path.abspath(os.curdir), 'buildout.cfg') + path = get_example_dir('buildout_project', 'src', 'proj_name') + parent = _get_parent_dir_with_file(path, 'buildout.cfg') assert parent is not None assert parent.endswith(os.path.join('test', 'examples', 'buildout_project')) -@cwd_at('test/examples/buildout_project/src/proj_name') def test_buildout_detection(Script): - scripts = list(_get_buildout_script_paths(os.path.abspath('./module_name.py'))) + path = get_example_dir('buildout_project', 'src', 'proj_name') + scripts = list(_get_buildout_script_paths(os.path.join(path, 'module_name.py'))) assert len(scripts) == 1 - curdir = os.path.abspath(os.curdir) - appdir_path = os.path.normpath(os.path.join(curdir, '../../bin/app')) + appdir_path = os.path.normpath(os.path.join(path, '../../bin/app')) assert scripts[0] == appdir_path @@ -53,13 +51,12 @@ def test_path_from_invalid_sys_path_assignment(Script): assert 'invalid' not in paths -@cwd_at('test/examples/buildout_project/src/proj_name/') def test_sys_path_with_modifications(Script): + path = get_example_dir('buildout_project', 'src', 'proj_name', 'module_name.py') code = dedent(""" import os """) - path = os.path.abspath(os.path.join(os.curdir, 'module_name.py')) paths = Script(code, path=path)._inference_state.get_sys_path() assert '/tmp/.buildout/eggs/important_package.egg' in paths diff --git a/test/test_inference/test_imports.py b/test/test_inference/test_imports.py index 5661371f..963e29a0 100644 --- a/test/test_inference/test_imports.py +++ b/test/test_inference/test_imports.py @@ -6,15 +6,15 @@ Tests". import os import pytest -from jedi.file_io import FileIO, KnownContentFileIO +from jedi.file_io import FileIO, KnownContentFileIO from jedi._compatibility import find_module_py33, find_module from jedi.inference import compiled from jedi.inference import imports from jedi.api.project import Project from jedi.inference.gradual.conversion import _stub_to_python_value_set from jedi.inference.references import get_module_contexts_containing_name -from ..helpers import cwd_at, get_example_dir, test_dir, root_dir +from ..helpers import get_example_dir, test_dir, test_dir_project, root_dir THIS_DIR = os.path.dirname(__file__) @@ -112,19 +112,24 @@ def test_find_module_not_package_zipped(Script, inference_state, environment): assert is_package is False -@cwd_at('test/examples/not_in_sys_path/pkg') -def test_import_not_in_sys_path(Script): +def test_import_not_in_sys_path(Script, environment): """ non-direct imports (not in sys.path) This is in the end just a fallback. """ - a = Script(path='module.py').infer(line=5) + path = get_example_dir() + module_path = os.path.join(path, 'not_in_sys_path', 'pkg', 'module.py') + # This project tests the smart path option of Project. The sys_path is + # explicitly given to make sure that the path is just dumb and only + # includes non-folder dependencies. + project = Project(path, sys_path=environment.get_sys_path()) + a = Script(path=module_path, project=project).infer(line=5) assert a[0].name == 'int' - a = Script(path='module.py').infer(line=6) + a = Script(path=module_path, project=project).infer(line=6) assert a[0].name == 'str' - a = Script(path='module.py').infer(line=7) + a = Script(path=module_path, project=project).infer(line=7) assert a[0].name == 'str' @@ -150,10 +155,9 @@ def test_flask_ext(Script, code, name): assert name in [c.name for c in completions] -@cwd_at('test/test_inference/') def test_not_importable_file(Script): src = 'import not_importable_file as x; x.' - assert not Script(src, path='example.py').complete() + assert not Script(src, path='example.py', project=test_dir_project).complete() def test_import_unique(Script): @@ -200,29 +204,28 @@ def test_goto_definition_on_import(Script): assert len(Script("import sys").infer(1, 8)) == 1 -@cwd_at('jedi') -def test_complete_on_empty_import(Script): - assert Script("from datetime import").complete()[0].name == 'import' +def test_complete_on_empty_import(ScriptWithProject): + assert ScriptWithProject("from datetime import").complete()[0].name == 'import' # should just list the files in the directory - assert 10 < len(Script("from .", path='whatever.py').complete()) < 30 + assert 10 < len(ScriptWithProject("from .", path='whatever.py').complete()) < 30 # Global import - assert len(Script("from . import", 'whatever.py').complete(1, 5)) > 30 + assert len(ScriptWithProject("from . import", 'whatever.py').complete(1, 5)) > 30 # relative import - assert 10 < len(Script("from . import", 'whatever.py').complete(1, 6)) < 30 + assert 10 < len(ScriptWithProject("from . import", 'whatever.py').complete(1, 6)) < 30 # Global import - assert len(Script("from . import classes", 'whatever.py').complete(1, 5)) > 30 + assert len(ScriptWithProject("from . import classes", 'whatever.py').complete(1, 5)) > 30 # relative import - assert 10 < len(Script("from . import classes", 'whatever.py').complete(1, 6)) < 30 + assert 10 < len(ScriptWithProject("from . import classes", 'whatever.py').complete(1, 6)) < 30 wanted = {'ImportError', 'import', 'ImportWarning'} - assert {c.name for c in Script("import").complete()} == wanted - assert len(Script("import import", path='').complete()) > 0 + assert {c.name for c in ScriptWithProject("import").complete()} == wanted + assert len(ScriptWithProject("import import", path='').complete()) > 0 # 111 - assert Script("from datetime import").complete()[0].name == 'import' - assert Script("from datetime import ").complete() + assert ScriptWithProject("from datetime import").complete()[0].name == 'import' + assert ScriptWithProject("from datetime import ").complete() def test_imports_on_global_namespace_without_path(Script): @@ -394,9 +397,9 @@ def test_relative_imports_with_outside_paths(Script): assert not script.complete() -@cwd_at('test/examples/issue1209/api/whatever/') def test_relative_imports_without_path(Script): - project = Project('.', sys_path=[], smart_sys_path=False) + path = get_example_dir('issue1209', 'api', 'whatever') + project = Project(path, sys_path=[], smart_sys_path=False) script = Script("from . ", project=project) assert [c.name for c in script.complete()] == ['api_test1', 'import'] diff --git a/test/test_speed.py b/test/test_speed.py index f08f0644..ea37a154 100644 --- a/test/test_speed.py +++ b/test/test_speed.py @@ -6,7 +6,7 @@ should. import time import functools -from .helpers import cwd_at, get_example_dir +from .helpers import get_example_dir import jedi @@ -44,7 +44,6 @@ def test_scipy_speed(Script): @_check_speed(0.8) -@cwd_at('test') def test_precedence_slowdown(Script): """ Precedence calculation can slow down things significantly in edge diff --git a/test/test_utils.py b/test/test_utils.py index a4c510e0..e91eb62d 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -5,7 +5,7 @@ except ImportError: from jedi import utils -from .helpers import unittest, cwd_at +from .helpers import unittest @unittest.skipIf(not readline, "readline not found") @@ -86,9 +86,8 @@ class TestSetupReadline(unittest.TestCase): # (posix and nt) librariesare included. assert len(difference) < 20 - @cwd_at('test') def test_local_import(self): - s = 'import test_utils' + s = 'import test.test_utils' assert self.complete(s) == [s] def test_preexisting_values(self): From f6465c5202f29926d9dd221412511b592f47f86c Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 6 Feb 2020 01:51:10 +0100 Subject: [PATCH 18/20] Get rid of one more os.getcwd() call --- jedi/api/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/api/__init__.py b/jedi/api/__init__.py index e33cef13..6630ead5 100644 --- a/jedi/api/__init__.py +++ b/jedi/api/__init__.py @@ -107,7 +107,7 @@ class Script(object): if project is None: # Load the Python grammar of the current interpreter. project = get_default_project( - os.path.dirname(self.path)if path else os.getcwd() + os.path.dirname(self.path) if path else None ) # TODO deprecate and remove sys_path from the Script API. if sys_path is not None: From 841fe75326a75c203300ae549af600d73e3ce940 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 6 Feb 2020 22:41:11 +0100 Subject: [PATCH 19/20] Fix an issue with environment selection --- jedi/api/project.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/jedi/api/project.py b/jedi/api/project.py index 9b418e2a..45e4c09a 100644 --- a/jedi/api/project.py +++ b/jedi/api/project.py @@ -91,7 +91,7 @@ class Project(object): @inference_state_as_method_param_cache() def _get_base_sys_path(self, inference_state): # The sys path has not been set explicitly. - sys_path = list(self.get_environment().get_sys_path()) + sys_path = list(inference_state.environment.get_sys_path()) try: sys_path.remove('') except ValueError: @@ -99,8 +99,7 @@ class Project(object): return sys_path @inference_state_as_method_param_cache() - def _get_sys_path(self, inference_state, environment=None, - add_parent_paths=True, add_init_paths=False): + def _get_sys_path(self, inference_state, add_parent_paths=True, add_init_paths=False): """ Keep this method private for all users of jedi. However internally this one is used like a public method. From 6e63799a7da32dda71cdacf8298cf272d08a546d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Thu, 6 Feb 2020 22:51:40 +0100 Subject: [PATCH 20/20] Fix a test that picked up the wrong paths --- test/test_inference/test_imports.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test/test_inference/test_imports.py b/test/test_inference/test_imports.py index 963e29a0..77b07be1 100644 --- a/test/test_inference/test_imports.py +++ b/test/test_inference/test_imports.py @@ -205,23 +205,24 @@ def test_goto_definition_on_import(Script): def test_complete_on_empty_import(ScriptWithProject): + path = os.path.join(test_dir, 'whatever.py') assert ScriptWithProject("from datetime import").complete()[0].name == 'import' # should just list the files in the directory - assert 10 < len(ScriptWithProject("from .", path='whatever.py').complete()) < 30 + assert 10 < len(ScriptWithProject("from .", path=path).complete()) < 30 # Global import - assert len(ScriptWithProject("from . import", 'whatever.py').complete(1, 5)) > 30 + assert len(ScriptWithProject("from . import", path=path).complete(1, 5)) > 30 # relative import - assert 10 < len(ScriptWithProject("from . import", 'whatever.py').complete(1, 6)) < 30 + assert 10 < len(ScriptWithProject("from . import", path=path).complete(1, 6)) < 30 # Global import - assert len(ScriptWithProject("from . import classes", 'whatever.py').complete(1, 5)) > 30 + assert len(ScriptWithProject("from . import classes", path=path).complete(1, 5)) > 30 # relative import - assert 10 < len(ScriptWithProject("from . import classes", 'whatever.py').complete(1, 6)) < 30 + assert 10 < len(ScriptWithProject("from . import classes", path=path).complete(1, 6)) < 30 wanted = {'ImportError', 'import', 'ImportWarning'} assert {c.name for c in ScriptWithProject("import").complete()} == wanted - assert len(ScriptWithProject("import import", path='').complete()) > 0 + assert len(ScriptWithProject("import import", path=path).complete()) > 0 # 111 assert ScriptWithProject("from datetime import").complete()[0].name == 'import'