Make it possible to get stdlib modules for project search

jedi/api/completion.py

@@ -576,11 +576,15 @@ def _complete_getattr(user_context, instance):
 
 
 def search_in_module(inference_state, module_context, names, wanted_names,
-                     wanted_type, complete=False, fuzzy=False):
+                     wanted_type, complete=False, fuzzy=False,
+                     ignore_imports=False):
     for s in wanted_names[:-1]:
         new_names = []
         for n in names:
             if s == n.string_name:
+                if n.tree_name is not None and n.api_type == 'module' \
+                        and ignore_imports:
+                    continue
                 new_names += complete_trailer(
                     module_context,
                     n.infer()

jedi/api/project.py

@@ -3,19 +3,21 @@ import errno
 import json
 import sys
 
-from jedi._compatibility import FileNotFoundError, PermissionError, IsADirectoryError
-from jedi._compatibility import scandir
+from jedi._compatibility import FileNotFoundError, PermissionError, \
+    IsADirectoryError, scandir
 from jedi.api.environment import get_cached_default_environment, create_environment
 from jedi.api.exceptions import WrongVersion
 from jedi.api.completion import search_in_module
 from jedi.api.helpers import split_search_string, get_module_names
 from jedi._compatibility import force_unicode
-from jedi.inference.imports import load_module_from_path, load_namespace_from_path
+from jedi.inference.imports import load_module_from_path, \
+    load_namespace_from_path, iter_module_names
 from jedi.inference.sys_path import discover_buildout_paths
 from jedi.inference.cache import inference_state_as_method_param_cache
 from jedi.inference.references import recurse_find_python_folders_and_files, search_in_file_ios
+from jedi.inference.value.module import ModuleValue
 from jedi.inference import InferenceState
-from jedi.file_io import FolderIO
+from jedi.file_io import FolderIO, FileIO
 from jedi.common.utils import traverse_parents
 
 _CONFIG_FOLDER = '.jedi'
@@ -177,7 +179,12 @@ class Project(object):
         """
         Returns a generator of names
         """
-        inference_state = InferenceState(self)
+        # Using a Script is they easiest way to get an empty module context.
+        from jedi import Script
+        s = Script('', project=self)
+        inference_state = s._inference_state
+        empty_module_context = s._get_module_context()
+
         if inference_state.grammar.version_info < (3, 6) or sys.version_info < (3, 6):
             raise NotImplementedError(
                 "No support for refactorings/search on Python 2/3.5"
@@ -188,6 +195,7 @@ class Project(object):
         ios = recurse_find_python_folders_and_files(FolderIO(self._path))
         file_ios = []
 
+        # 1. Search for modules in the current project
         for folder_io, file_io in ios:
             if file_io is None:
                 file_name = folder_io.get_base_name()
@@ -221,6 +229,7 @@ class Project(object):
             ):
                 yield x  # Python 2...
 
+        # 2. Search for identifiers in the project.
         for module_context in search_in_file_ios(inference_state, file_ios, name):
             names = get_module_names(module_context.tree_node, all_scopes=all_scopes)
             for x in search_in_module(
@@ -229,6 +238,26 @@ class Project(object):
                 names=[module_context.create_name(n) for n in names],
                 wanted_type=wanted_type,
                 wanted_names=wanted_names,
+                complete=complete,
+                ignore_imports=True,
+            ):
+                yield x  # Python 2...
+
+        # 3. Search for modules on sys.path
+        sys_path = [
+            p for p in self._get_sys_path(inference_state)
+            # Exclude folders that are handled by recursing of the Python
+            # folders.
+            if not p.startswith(self._path)
+        ]
+
+        names = list(iter_module_names(inference_state, empty_module_context, sys_path))
+        for x in search_in_module(
+            inference_state,
+            empty_module_context,
+            names=names,
+            wanted_type=wanted_type,
+            wanted_names=wanted_names,
             complete=complete
         ):
             yield x  # Python 2...
@@ -295,11 +324,10 @@ def get_default_project(path=None):
     return Project(curdir)
 
 
-def _recursive_file_list(path):
-    listed = sorted(scandir(path), key=lambda e: e.name)
-    for entry in listed:
-        if entry.is_dir(follow_symlinks=True):
-            for x in _recursive_file_list(entry.path):  # Python 2...
-                yield x
+def _get_sys_path_folder_and_file_ios(sys_path):
+    for p in sys_path:
+        for dir_entry in scandir(p):
+            if dir_entry.is_dir():
+                yield FolderIO(p), None
             else:
-            yield entry
+                yield None, FileIO(p)

jedi/inference/imports.py

@@ -29,7 +29,7 @@ from jedi.inference.cache import inference_state_method_cache
 from jedi.inference.names import ImportName, SubModuleName
 from jedi.inference.base_value import ValueSet, NO_VALUES
 from jedi.inference.gradual.typeshed import import_module_decorator
-from jedi.inference.value.module import iter_module_names
+from jedi.inference.value.module import iter_module_names as module_iter_module_names
 from jedi.plugins import plugin_manager
 
 
@@ -265,24 +265,15 @@ class Importer(object):
         Get the names of all modules in the search_path. This means file names
         and not names defined in the files.
         """
-        names = []
-        # add builtin module names
-        if search_path is None and in_module is None:
-            names += [
-                ImportName(self._module_context, name)
-                for name in self._inference_state.compiled_subprocess.get_builtin_module_names()
-            ]
-
         if search_path is None:
-            search_path = self._sys_path_with_modifications(is_completion=True)
-
-        for name in iter_module_names(self._inference_state, search_path):
-            if in_module is None:
-                n = ImportName(self._module_context, name)
+            sys_path = self._sys_path_with_modifications(is_completion=True)
         else:
-                n = SubModuleName(in_module.as_context(), name)
-            names.append(n)
-        return names
+            sys_path = search_path
+        return list(iter_module_names(
+            self._inference_state, self._module_context, sys_path,
+            module_cls=ImportName if in_module is None else SubModuleName,
+            add_builtin_modules=search_path is None and in_module is None,
+        ))
 
     def completion_names(self, inference_state, only_modules=False):
         """
@@ -536,3 +527,18 @@ def follow_error_node_imports_if_possible(context, name):
             return Importer(
                 context.inference_state, names, context.get_root_context(), level).follow()
     return None
+
+
+def iter_module_names(inference_state, module_context, search_path,
+                      module_cls=ImportName, add_builtin_modules=True):
+    """
+    Get the names of all modules in the search_path. This means file names
+    and not names defined in the files.
+    """
+    # add builtin module names
+    if add_builtin_modules:
+        for name in inference_state.compiled_subprocess.get_builtin_module_names():
+            yield module_cls(module_context, name)
+
+    for name in module_iter_module_names(inference_state, search_path):
+        yield module_cls(module_context, name)

test/test_api/test_project.py

@@ -93,6 +93,10 @@ def test_load_save_project(tmpdir):
           'examples.implicit_namespace_package.ns1',
           'examples.implicit_namespace_package.ns2'],
          dict(complete=True)),
+
+        # On sys path
+        ('sys.path', ['sys.path'], {}),
+        ('json.dumps', ['json.dumps', 'json.dumps'], {}),  # stdlib + stub
     ]
 )
 @pytest.mark.skipif(sys.version_info < (3, 6), reason="Ignore Python 2, because EOL")