Typeshed third party libraries should not be loaded if they don't actually exist in the environment, fixes #1620

jedi/inference/gradual/type_var.py

@@ -102,7 +102,7 @@ class TypeVar(BaseTypingValue):
         else:
             if found:
                 return found
-        return self._get_classes() or ValueSet({self})
+        return ValueSet({self})
 
     def execute_annotation(self):
         return self._get_classes().execute_annotation()

jedi/inference/gradual/typeshed.py

@@ -1,6 +1,7 @@
 import os
 import re
 from functools import wraps
+from collections import namedtuple
 
 from jedi import settings
 from jedi.file_io import FileIO
@@ -20,36 +21,38 @@ _IMPORT_MAP = dict(
     _socket='socket',
 )
 
+PathInfo = namedtuple('PathInfo', 'path is_third_party')
 
-def _merge_create_stub_map(directories):
+
+def _merge_create_stub_map(path_infos):
     map_ = {}
-    for directory in directories:
+    for directory_path_info in path_infos:
-        map_.update(_create_stub_map(directory))
+        map_.update(_create_stub_map(directory_path_info))
     return map_
 
 
-def _create_stub_map(directory):
+def _create_stub_map(directory_path_info):
     """
     Create a mapping of an importable name in Python to a stub file.
     """
     def generate():
         try:
-            listed = os.listdir(directory)
+            listed = os.listdir(directory_path_info.path)
         except (FileNotFoundError, OSError):
             # OSError is Python 2
             return
 
         for entry in listed:
             entry = cast_path(entry)
-            path = os.path.join(directory, entry)
+            path = os.path.join(directory_path_info.path, entry)
             if os.path.isdir(path):
                 init = os.path.join(path, '__init__.pyi')
                 if os.path.isfile(init):
-                    yield entry, init
+                    yield entry, PathInfo(init, directory_path_info.is_third_party)
             elif entry.endswith('.pyi') and os.path.isfile(path):
                 name = entry[:-4]
                 if name != '__init__':
-                    yield name, path
+                    yield name, PathInfo(path, directory_path_info.is_third_party)
 
     # Create a dictionary from the tuple generator.
     return dict(generate())
@@ -58,8 +61,8 @@ def _create_stub_map(directory):
 def _get_typeshed_directories(version_info):
     check_version_list = ['2and3', str(version_info.major)]
     for base in ['stdlib', 'third_party']:
-        base = os.path.join(TYPESHED_PATH, base)
+        base_path = os.path.join(TYPESHED_PATH, base)
-        base_list = os.listdir(base)
+        base_list = os.listdir(base_path)
         for base_list_entry in base_list:
             match = re.match(r'(\d+)\.(\d+)$', base_list_entry)
             if match is not None:
@@ -68,7 +71,8 @@ def _get_typeshed_directories(version_info):
                     check_version_list.append(base_list_entry)
 
         for check_version in check_version_list:
-            yield os.path.join(base, check_version)
+            is_third_party = base != 'stdlib'
+            yield PathInfo(os.path.join(base_path, check_version), is_third_party)
 
 
 _version_cache = {}
@@ -175,7 +179,7 @@ def _try_to_load_stub(inference_state, import_names, python_value_set,
             )
             if m is not None:
                 return m
-        if import_names[0] == 'django':
+        if import_names[0] == 'django' and python_value_set:
             return _try_to_load_stub_from_file(
                 inference_state,
                 python_value_set,
@@ -249,16 +253,21 @@ def _load_from_typeshed(inference_state, python_value_set, parent_module_value,
             # Only if it's a package (= a folder) something can be
             # imported.
             return None
-        path = parent_module_value.py__path__()
+        paths = parent_module_value.py__path__()
-        map_ = _merge_create_stub_map(path)
+        # Once the initial package has been loaded, the sub packages will
+        # always be loaded, regardless if they are there or not. This makes
+        # sense, IMO, because stubs take preference, even if the original
+        # library doesn't provide a module (it could be dynamic). ~dave
+        map_ = _merge_create_stub_map([PathInfo(p, is_third_party=False) for p in paths])
 
     if map_ is not None:
-        path = map_.get(import_name)
+        path_info = map_.get(import_name)
-        if path is not None:
+        print(path_info)
+        if path_info is not None and (not path_info.is_third_party or python_value_set):
             return _try_to_load_stub_from_file(
                 inference_state,
                 python_value_set,
-                file_io=FileIO(path),
+                file_io=FileIO(path_info.path),
                 import_names=import_names,
             )
 

test/test_inference/test_gradual/test_typeshed.py

@@ -14,8 +14,8 @@ TYPESHED_PYTHON3 = os.path.join(typeshed.TYPESHED_PATH, 'stdlib', '3')
 def test_get_typeshed_directories():
     def get_dirs(version_info):
         return {
-            d.replace(typeshed.TYPESHED_PATH, '').lstrip(os.path.sep)
+            p.path.replace(typeshed.TYPESHED_PATH, '').lstrip(os.path.sep)
-            for d in typeshed._get_typeshed_directories(version_info)
+            for p in typeshed._get_typeshed_directories(version_info)
         }
 
     def transform(set_):
@@ -35,11 +35,8 @@ def test_get_typeshed_directories():
 
 
 def test_get_stub_files():
-    def get_map(version_info):
+    map_ = typeshed._create_stub_map(typeshed.PathInfo(TYPESHED_PYTHON3, is_third_party=False))
-        return typeshed._create_stub_map(version_info)
+    assert map_['functools'].path == os.path.join(TYPESHED_PYTHON3, 'functools.pyi')
-
-    map_ = typeshed._create_stub_map(TYPESHED_PYTHON3)
-    assert map_['functools'] == os.path.join(TYPESHED_PYTHON3, 'functools.pyi')
 
 
 def test_function(Script, environment):
@@ -227,3 +224,25 @@ def test_goto_stubs_on_itself(Script, code, type_):
 
     _assert_is_same(same_definition, definition)
     _assert_is_same(same_definition, same_definition2)
+
+
+def test_module_exists_only_as_stub(Script):
+    try:
+        import redis
+    except ImportError:
+        pass
+    else:
+        pytest.skip('redis is already installed, it should only exist as a stub for this test')
+    redis_path = os.path.join(typeshed.TYPESHED_PATH, 'third_party', '2and3', 'redis')
+    assert os.path.isdir(redis_path)
+    assert not Script('import redis').infer()
+
+
+def test_django_exists_only_as_stub(Script):
+    try:
+        import django
+    except ImportError:
+        pass
+    else:
+        pytest.skip('django is already installed, it should only exist as a stub for this test')
+    assert not Script('import django').infer()