From 06bb3cd50bf58dc5d59eb9bfdad87993888c9bc7 Mon Sep 17 00:00:00 2001 From: Maxim Kurnikov Date: Sun, 18 Nov 2018 15:58:17 +0300 Subject: [PATCH] add support for django.conf.settings.SETTING_NAME --- conftest.py | 2 +- django-stubs/conf/__init__.pyi | 12 +- django-stubs/utils/functional.pyi | 57 +++++ mypy_django_plugin/helpers.py | 20 +- mypy_django_plugin/{plugin.py => main.py} | 15 ++ mypy_django_plugin/plugins/setup_settings.py | 34 +++ test/__init__.py | 0 test/data.py | 244 +++++++++++++++++++ test/plugins.ini | 2 +- test/test-data/check-parse-settings.test | 31 +++ test/testdjango.py | 3 +- 11 files changed, 412 insertions(+), 8 deletions(-) create mode 100644 django-stubs/utils/functional.pyi rename mypy_django_plugin/{plugin.py => main.py} (71%) create mode 100644 mypy_django_plugin/plugins/setup_settings.py create mode 100644 test/__init__.py create mode 100644 test/data.py create mode 100644 test/test-data/check-parse-settings.test diff --git a/conftest.py b/conftest.py index d8b9cf8..8b7b61b 100644 --- a/conftest.py +++ b/conftest.py @@ -1,3 +1,3 @@ pytest_plugins = [ - 'mypy.test.data' + 'test.data' ] \ No newline at end of file diff --git a/django-stubs/conf/__init__.pyi b/django-stubs/conf/__init__.pyi index 03265d8..dec9bb1 100644 --- a/django-stubs/conf/__init__.pyi +++ b/django-stubs/conf/__init__.pyi @@ -1,4 +1,14 @@ from typing import Any +from django.utils.functional import LazyObject -settings = ... # type: Any + +# required for plugin to be able to distinguish this specific instance of LazySettings from others +class _DjangoConfLazyObject(LazyObject): ... + + +class LazySettings(_DjangoConfLazyObject): + configured: bool + def configure(self, default_settings: Any = ..., **options: Any) -> Any: ... + +settings: LazySettings = ... \ No newline at end of file diff --git a/django-stubs/utils/functional.pyi b/django-stubs/utils/functional.pyi new file mode 100644 index 0000000..3a78ec4 --- /dev/null +++ b/django-stubs/utils/functional.pyi @@ -0,0 +1,57 @@ +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union + +from django.db.models.base import Model + + +def curry(_curried_func: Any, *args: Any, **kwargs: Any): ... + +class cached_property: + func: Callable = ... + __doc__: Any = ... + name: str = ... + def __init__(self, func: Callable, name: Optional[str] = ...) -> None: ... + def __get__(self, instance: Any, cls: Type[Any] = ...) -> Any: ... + +class Promise: ... + +def lazy(func: Union[Callable, Type[str]], *resultclasses: Any) -> Callable: ... +def lazystr(text: Any): ... +def keep_lazy(*resultclasses: Any) -> Callable: ... +def keep_lazy_text(func: Callable) -> Callable: ... + +empty: Any + +def new_method_proxy(func: Any): ... + +class LazyObject: + def __init__(self) -> None: ... + __getattr__: Any = ... + def __setattr__(self, name: str, value: Any) -> None: ... + def __delattr__(self, name: str) -> None: ... + def __reduce__(self) -> Tuple[Callable, Tuple[Model]]: ... + def __copy__(self): ... + def __deepcopy__(self, memo: Any): ... + __bytes__: Any = ... + __bool__: Any = ... + __dir__: Any = ... + __class__: Any = ... + __eq__: Any = ... + __ne__: Any = ... + __hash__: Any = ... + __getitem__: Any = ... + __setitem__: Any = ... + __delitem__: Any = ... + __iter__: Any = ... + __len__: Any = ... + __contains__: Any = ... + +def unpickle_lazyobject(wrapped: Model) -> Model: ... + +class SimpleLazyObject(LazyObject): + def __init__(self, func: Callable) -> None: ... + def __copy__(self) -> List[int]: ... + def __deepcopy__(self, memo: Dict[Any, Any]) -> List[int]: ... + +def partition( + predicate: Callable, values: List[Model] +) -> Tuple[List[Model], List[Model]]: ... diff --git a/mypy_django_plugin/helpers.py b/mypy_django_plugin/helpers.py index a844291..13402fe 100644 --- a/mypy_django_plugin/helpers.py +++ b/mypy_django_plugin/helpers.py @@ -1,8 +1,10 @@ -from typing import Dict, Optional, Type, Tuple, NamedTuple +from typing import Dict, Optional, NamedTuple -from mypy.nodes import SymbolTableNode, Var, Expression, MemberExpr +from mypy.semanal import SemanticAnalyzerPass2 +from mypy.types import Type +from mypy.nodes import SymbolTableNode, Var, Expression from mypy.plugin import FunctionContext -from mypy.types import Instance +from mypy.types import Instance, UnionType, NoneTyp MODEL_CLASS_FULLNAME = 'django.db.models.base.Model' QUERYSET_CLASS_FULLNAME = 'django.db.models.query.QuerySet' @@ -13,7 +15,6 @@ ONETOONE_FIELD_FULLNAME = 'django.db.models.fields.related.OneToOneField' def create_new_symtable_node(name: str, kind: int, instance: Instance) -> SymbolTableNode: new_var = Var(name, instance) new_var.info = instance.type - return SymbolTableNode(kind, new_var, plugin_generated=True) @@ -53,3 +54,14 @@ def get_call_signature_or_none(ctx: FunctionContext) -> Optional[Dict[str, Argum result[arg_name] = (arg[0], arg_type[0]) return result + + +def make_optional(typ: Type) -> Type: + return UnionType.make_simplified_union([typ, NoneTyp()]) + + +def make_required(typ: Type) -> Type: + if not isinstance(typ, UnionType): + return typ + items = [item for item in typ.items if not isinstance(item, NoneTyp)] + return UnionType.make_union(items) \ No newline at end of file diff --git a/mypy_django_plugin/plugin.py b/mypy_django_plugin/main.py similarity index 71% rename from mypy_django_plugin/plugin.py rename to mypy_django_plugin/main.py index ec5f34a..2227cf9 100644 --- a/mypy_django_plugin/plugin.py +++ b/mypy_django_plugin/main.py @@ -1,5 +1,8 @@ +import os from typing import Callable, Optional +from django.conf import Settings +from mypy.options import Options from mypy.plugin import Plugin, FunctionContext, ClassDefContext from mypy.types import Type @@ -8,6 +11,7 @@ from mypy_django_plugin.plugins.objects_queryset import set_objects_queryset_to_ from mypy_django_plugin.plugins.postgres_fields import determine_type_of_array_field from mypy_django_plugin.plugins.related_fields import set_related_name_instance_for_onetoonefield, \ set_related_name_manager_for_foreign_key, set_fieldname_attrs_for_related_fields +from mypy_django_plugin.plugins.setup_settings import DjangoConfSettingsInitializerHook base_model_classes = {helpers.MODEL_CLASS_FULLNAME} @@ -21,6 +25,15 @@ def transform_model_class(ctx: ClassDefContext) -> None: class DjangoPlugin(Plugin): + def __init__(self, + options: Options) -> None: + super().__init__(options) + self.django_settings = None + + django_settings_module = os.environ.get('DJANGO_SETTINGS_MODULE') + if django_settings_module: + self.django_settings = Settings(django_settings_module) + def get_function_hook(self, fullname: str ) -> Optional[Callable[[FunctionContext], Type]]: if fullname == helpers.FOREIGN_KEY_FULLNAME: @@ -37,6 +50,8 @@ class DjangoPlugin(Plugin): ) -> Optional[Callable[[ClassDefContext], None]]: if fullname in base_model_classes: return transform_model_class + if fullname == 'django.conf._DjangoConfLazyObject': + return DjangoConfSettingsInitializerHook(settings=self.django_settings) return None diff --git a/mypy_django_plugin/plugins/setup_settings.py b/mypy_django_plugin/plugins/setup_settings.py new file mode 100644 index 0000000..208a26e --- /dev/null +++ b/mypy_django_plugin/plugins/setup_settings.py @@ -0,0 +1,34 @@ +from typing import cast, Any + +from django.conf import Settings +from mypy.nodes import MDEF, TypeInfo, SymbolTable +from mypy.plugin import ClassDefContext +from mypy.semanal import SemanticAnalyzerPass2 +from mypy.types import Instance, AnyType, TypeOfAny + +from mypy_django_plugin import helpers + + +def get_obj_type_name(value: Any) -> str: + return type(value).__module__ + '.' + type(value).__qualname__ + + +class DjangoConfSettingsInitializerHook(object): + def __init__(self, settings: Settings): + self.settings = settings + + def __call__(self, ctx: ClassDefContext) -> None: + api = cast(SemanticAnalyzerPass2, ctx.api) + for name, value in self.settings.__dict__.items(): + if name.isupper(): + if value is None: + ctx.cls.info.names[name] = helpers.create_new_symtable_node(name, MDEF, + instance=api.builtin_type('builtins.object')) + continue + + type_fullname = get_obj_type_name(value) + sym = api.lookup_fully_qualified_or_none(type_fullname) + if sym is not None: + args = len(sym.node.type_vars) * [AnyType(TypeOfAny.from_omitted_generics)] + ctx.cls.info.names[name] = helpers.create_new_symtable_node(name, MDEF, + instance=Instance(sym.node, args)) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/data.py b/test/data.py new file mode 100644 index 0000000..b4ba267 --- /dev/null +++ b/test/data.py @@ -0,0 +1,244 @@ +import os +import posixpath +import re +import sys +import tempfile +from typing import Any, Optional, Iterator, Dict, List, Tuple, Set + +import pytest +from mypy.test.config import test_temp_dir +from mypy.test.data import DataDrivenTestCase, DataSuite, add_test_name_suffix, parse_test_data, \ + expand_errors, expand_variables, fix_win_path + + +def parse_test_case(case: 'DataDrivenTestCase') -> None: + """Parse and prepare a single case from suite with test case descriptions. + + This method is part of the setup phase, just before the test case is run. + """ + test_items = parse_test_data(case.data, case.name) + base_path = case.suite.base_path + if case.suite.native_sep: + join = os.path.join + else: + join = posixpath.join # type: ignore + + out_section_missing = case.suite.required_out_section + + files = [] # type: List[Tuple[str, str]] # path and contents + output_files = [] # type: List[Tuple[str, str]] # path and contents for output files + output = [] # type: List[str] # Regular output errors + output2 = {} # type: Dict[int, List[str]] # Output errors for incremental, runs 2+ + deleted_paths = {} # type: Dict[int, Set[str]] # from run number of paths + stale_modules = {} # type: Dict[int, Set[str]] # from run number to module names + rechecked_modules = {} # type: Dict[ int, Set[str]] # from run number module names + triggered = [] # type: List[str] # Active triggers (one line per incremental step) + + # Process the parsed items. Each item has a header of form [id args], + # optionally followed by lines of text. + item = first_item = test_items[0] + for item in test_items[1:]: + if item.id == 'file' or item.id == 'outfile': + # Record an extra file needed for the test case. + assert item.arg is not None + contents = expand_variables('\n'.join(item.data)) + file_entry = (join(base_path, item.arg), contents) + if item.id == 'file': + files.append(file_entry) + else: + output_files.append(file_entry) + elif item.id in ('builtins', 'builtins_py2'): + # Use an alternative stub file for the builtins module. + assert item.arg is not None + mpath = join(os.path.dirname(case.file), item.arg) + fnam = 'builtins.pyi' if item.id == 'builtins' else '__builtin__.pyi' + with open(mpath) as f: + files.append((join(base_path, fnam), f.read())) + elif item.id == 'typing': + # Use an alternative stub file for the typing module. + assert item.arg is not None + src_path = join(os.path.dirname(case.file), item.arg) + with open(src_path) as f: + files.append((join(base_path, 'typing.pyi'), f.read())) + elif re.match(r'stale[0-9]*$', item.id): + passnum = 1 if item.id == 'stale' else int(item.id[len('stale'):]) + assert passnum > 0 + modules = (set() if item.arg is None else {t.strip() for t in item.arg.split(',')}) + stale_modules[passnum] = modules + elif re.match(r'rechecked[0-9]*$', item.id): + passnum = 1 if item.id == 'rechecked' else int(item.id[len('rechecked'):]) + assert passnum > 0 + modules = (set() if item.arg is None else {t.strip() for t in item.arg.split(',')}) + rechecked_modules[passnum] = modules + elif item.id == 'delete': + # File to delete during a multi-step test case + assert item.arg is not None + m = re.match(r'(.*)\.([0-9]+)$', item.arg) + assert m, 'Invalid delete section: {}'.format(item.arg) + num = int(m.group(2)) + assert num >= 2, "Can't delete during step {}".format(num) + full = join(base_path, m.group(1)) + deleted_paths.setdefault(num, set()).add(full) + elif re.match(r'out[0-9]*$', item.id): + tmp_output = [expand_variables(line) for line in item.data] + if os.path.sep == '\\': + tmp_output = [fix_win_path(line) for line in tmp_output] + if item.id == 'out' or item.id == 'out1': + output = tmp_output + else: + passnum = int(item.id[len('out'):]) + assert passnum > 1 + output2[passnum] = tmp_output + out_section_missing = False + elif item.id == 'triggered' and item.arg is None: + triggered = item.data + elif item.id == 'env': + env_vars_to_set = item.arg + for env in env_vars_to_set.split(';'): + try: + name, value = env.split('=') + os.environ[name] = value + except ValueError: + continue + else: + raise ValueError( + 'Invalid section header {} in {} at line {}'.format( + item.id, case.file, item.line)) + + if out_section_missing: + raise ValueError( + '{}, line {}: Required output section not found'.format( + case.file, first_item.line)) + + for passnum in stale_modules.keys(): + if passnum not in rechecked_modules: + # If the set of rechecked modules isn't specified, make it the same as the set + # of modules with a stale public interface. + rechecked_modules[passnum] = stale_modules[passnum] + if (passnum in stale_modules + and passnum in rechecked_modules + and not stale_modules[passnum].issubset(rechecked_modules[passnum])): + raise ValueError( + ('Stale modules after pass {} must be a subset of rechecked ' + 'modules ({}:{})').format(passnum, case.file, first_item.line)) + + input = first_item.data + expand_errors(input, output, 'main') + for file_path, contents in files: + expand_errors(contents.split('\n'), output, file_path) + + case.input = input + case.output = output + case.output2 = output2 + case.lastline = item.line + case.files = files + case.output_files = output_files + case.expected_stale_modules = stale_modules + case.expected_rechecked_modules = rechecked_modules + case.deleted_paths = deleted_paths + case.triggered = triggered or [] + + +class DjangoDataDrivenTestCase(DataDrivenTestCase): + def setup(self) -> None: + self.old_environ = os.environ.copy() + + parse_test_case(case=self) + self.old_cwd = os.getcwd() + + self.tmpdir = tempfile.TemporaryDirectory(prefix='mypy-test-') + + os.chdir(self.tmpdir.name) + os.mkdir(test_temp_dir) + encountered_files = set() + self.clean_up = [] + for paths in self.deleted_paths.values(): + for path in paths: + self.clean_up.append((False, path)) + encountered_files.add(path) + for path, content in self.files: + dir = os.path.dirname(path) + for d in self.add_dirs(dir): + self.clean_up.append((True, d)) + with open(path, 'w') as f: + f.write(content) + if path not in encountered_files: + self.clean_up.append((False, path)) + encountered_files.add(path) + if re.search(r'\.[2-9]$', path): + # Make sure new files introduced in the second and later runs are accounted for + renamed_path = path[:-2] + if renamed_path not in encountered_files: + encountered_files.add(renamed_path) + self.clean_up.append((False, renamed_path)) + for path, _ in self.output_files: + # Create directories for expected output and mark them to be cleaned up at the end + # of the test case. + dir = os.path.dirname(path) + for d in self.add_dirs(dir): + self.clean_up.append((True, d)) + self.clean_up.append((False, path)) + + sys.path.insert(0, os.path.join(self.tmpdir.name, 'tmp')) + + def teardown(self): + if hasattr(self, 'old_environ'): + os.environ = self.old_environ + super().teardown() + + +def split_test_cases(parent: 'DataSuiteCollector', suite: 'DataSuite', + file: str) -> Iterator[DjangoDataDrivenTestCase]: + """Iterate over raw test cases in file, at collection time, ignoring sub items. + + The collection phase is slow, so any heavy processing should be deferred to after + uninteresting tests are filtered (when using -k PATTERN switch). + """ + with open(file, encoding='utf-8') as f: + data = f.read() + cases = re.split(r'^\[case ([a-zA-Z_0-9]+)' + r'(-writescache)?' + r'(-only_when_cache|-only_when_nocache)?' + r'(-skip)?' + r'\][ \t]*$\n', data, + flags=re.DOTALL | re.MULTILINE) + line_no = cases[0].count('\n') + 1 + + for i in range(1, len(cases), 5): + name, writescache, only_when, skip, data = cases[i:i + 5] + yield DjangoDataDrivenTestCase(parent, suite, file, + name=add_test_name_suffix(name, suite.test_name_suffix), + writescache=bool(writescache), + only_when=only_when, + skip=bool(skip), + data=data, + line=line_no) + line_no += data.count('\n') + 1 + + +class DataSuiteCollector(pytest.Class): # type: ignore # inheriting from Any + def collect(self) -> Iterator[pytest.Item]: # type: ignore + """Called by pytest on each of the object returned from pytest_pycollect_makeitem""" + + # obj is the object for which pytest_pycollect_makeitem returned self. + suite = self.obj # type: DataSuite + for f in suite.files: + yield from split_test_cases(self, suite, os.path.join(suite.data_prefix, f)) + + +# This function name is special to pytest. See +# http://doc.pytest.org/en/latest/writing_plugins.html#collection-hooks +def pytest_pycollect_makeitem(collector: Any, name: str, + obj: object) -> 'Optional[Any]': + """Called by pytest on each object in modules configured in conftest.py files. + + collector is pytest.Collector, returns Optional[pytest.Class] + """ + if isinstance(obj, type): + # Only classes derived from DataSuite contain test cases, not the DataSuite class itself + if issubclass(obj, DataSuite) and obj is not DataSuite: + # Non-None result means this obj is a test case. + # The collect method of the returned DataSuiteCollector instance will be called later, + # with self.obj being obj. + return DataSuiteCollector(name, parent=collector) + return None diff --git a/test/plugins.ini b/test/plugins.ini index 409da66..b1b4489 100644 --- a/test/plugins.ini +++ b/test/plugins.ini @@ -1,3 +1,3 @@ [mypy] plugins = - mypy_django_plugin.plugin + mypy_django_plugin.main diff --git a/test/test-data/check-parse-settings.test b/test/test-data/check-parse-settings.test new file mode 100644 index 0000000..15e396f --- /dev/null +++ b/test/test-data/check-parse-settings.test @@ -0,0 +1,31 @@ +[case testParseSettingsFromFile] +from django.conf import settings + +reveal_type(settings.ROOT_DIR) # E: Revealed type is 'builtins.str' +reveal_type(settings.OBJ) # E: Revealed type is 'django.utils.functional.LazyObject' +reveal_type(settings.NUMBERS) # E: Revealed type is 'builtins.list[Any]' +reveal_type(settings.DICT) # E: Revealed type is 'builtins.dict[Any, Any]' + +[file mysettings.py] +SECRET_KEY = 112233 +ROOT_DIR = '/etc' +NUMBERS = ['one', 'two'] +DICT = {} + +from django.utils.functional import LazyObject +OBJ = LazyObject() + +[env DJANGO_SETTINGS_MODULE=mysettings] +[out] + +[case testSettingIsInitializedToNone] +from django.conf import settings + +reveal_type(settings.NONE_SETTING) # E: Revealed type is 'builtins.object' + +[file mysettings_not_none.py] +SECRET_KEY = 112233 +NONE_SETTING = None + +[env DJANGO_SETTINGS_MODULE=mysettings_not_none] +[out] \ No newline at end of file diff --git a/test/testdjango.py b/test/testdjango.py index f466dec..cd67f82 100644 --- a/test/testdjango.py +++ b/test/testdjango.py @@ -17,7 +17,8 @@ class DjangoTestSuite(DataSuite): 'check-objects-queryset.test', 'check-model-fields.test', 'check-postgres-fields.test', - 'check-model-relations.test' + 'check-model-relations.test', + 'check-parse-settings.test' ] data_prefix = str(TEST_DATA_DIR)