new semanal wip 1

This commit is contained in:
Maxim Kurnikov
2019-07-16 01:22:20 +03:00
parent 9c5a6be9a7
commit b11a9a85f9
96 changed files with 4441 additions and 2370 deletions

View File

@@ -0,0 +1,54 @@
import os
from configparser import ConfigParser
from typing import Dict, List, Optional
import dataclasses
from dataclasses import dataclass
from pytest_mypy.utils import temp_environ
@dataclass
class Config:
django_settings_module: Optional[str] = None
installed_apps: List[str] = dataclasses.field(default_factory=list)
ignore_missing_settings: bool = False
ignore_missing_model_attributes: bool = False
@classmethod
def from_config_file(cls, fpath: str) -> 'Config':
ini_config = ConfigParser()
ini_config.read(fpath)
if not ini_config.has_section('mypy_django_plugin'):
raise ValueError('Invalid config file: no [mypy_django_plugin] section')
django_settings = ini_config.get('mypy_django_plugin', 'django_settings',
fallback=None)
if django_settings:
django_settings = django_settings.strip()
return Config(django_settings_module=django_settings,
ignore_missing_settings=bool(ini_config.get('mypy_django_plugin',
'ignore_missing_settings',
fallback=False)),
ignore_missing_model_attributes=bool(ini_config.get('mypy_django_plugin',
'ignore_missing_model_attributes',
fallback=False)))
def extract_app_model_aliases(settings_module: str) -> Dict[str, str]:
with temp_environ():
os.environ['DJANGO_SETTINGS_MODULE'] = settings_module
import django
django.setup()
app_model_mapping: Dict[str, str] = {}
from django.apps import apps
for name, app_config in apps.app_configs.items():
app_label = app_config.label
for model_name, model_class in app_config.models.items():
app_model_mapping[app_label + '.' + model_class.__name__] = model_class.__module__ + '.' + model_class.__name__
return app_model_mapping

View File

@@ -26,4 +26,10 @@ MANAGER_CLASSES = {
RELATED_MANAGER_CLASS_FULLNAME,
BASE_MANAGER_CLASS_FULLNAME,
QUERYSET_CLASS_FULLNAME
}
RELATED_FIELDS_CLASSES = {
FOREIGN_KEY_FULLNAME,
ONETOONE_FIELD_FULLNAME,
MANYTOMANY_FIELD_FULLNAME
}

View File

@@ -1,24 +1,20 @@
import typing
from collections import OrderedDict
from typing import Dict, Optional, cast
from typing import Dict, Iterator, List, Optional, Set, TYPE_CHECKING, Tuple, Union, cast
from mypy.mro import calculate_mro
from mypy.nodes import (
GDEF, MDEF, AssignmentStmt, Block, CallExpr, ClassDef, Expression, ImportedName, Lvalue, MypyFile, NameExpr,
SymbolNode, SymbolTable, SymbolTableNode, TypeInfo, Var,
)
from mypy.nodes import (AssignmentStmt, Block, CallExpr, ClassDef, Expression, FakeInfo, GDEF, ImportedName, Lvalue, MDEF,
MemberExpr, MypyFile, NameExpr, SymbolNode, SymbolTable, SymbolTableNode, TypeInfo, Var)
from mypy.plugin import CheckerPluginInterface, FunctionContext, MethodContext
from mypy.types import (
AnyType, Instance, NoneTyp, TupleType, Type, TypedDictType, TypeOfAny, TypeVarType, UnionType,
)
from mypy.types import (AnyType, Instance, NoneTyp, TupleType, Type as MypyType, TypeOfAny, TypeVarType, TypedDictType,
UnionType)
from mypy_django_plugin.lib import metadata, fullnames
from mypy_django_plugin.lib import fullnames, metadata
if typing.TYPE_CHECKING:
if TYPE_CHECKING:
from mypy.checker import TypeChecker
def get_models_file(app_name: str, all_modules: typing.Dict[str, MypyFile]) -> Optional[MypyFile]:
def get_models_file(app_name: str, all_modules: Dict[str, MypyFile]) -> Optional[MypyFile]:
models_module = '.'.join([app_name, 'models'])
return all_modules.get(models_module)
@@ -85,7 +81,7 @@ def parse_bool(expr: Expression) -> Optional[bool]:
return None
def reparametrize_instance(instance: Instance, new_args: typing.List[Type]) -> Instance:
def reparametrize_instance(instance: Instance, new_args: List[MypyType]) -> Instance:
return Instance(instance.type, args=new_args,
line=instance.line, column=instance.column)
@@ -94,7 +90,7 @@ def fill_typevars_with_any(instance: Instance) -> Instance:
return reparametrize_instance(instance, [AnyType(TypeOfAny.unannotated)])
def extract_typevar_value(tp: Instance, typevar_name: str) -> Type:
def extract_typevar_value(tp: Instance, typevar_name: str) -> MypyType:
if typevar_name in {'_T', '_T_co'}:
if '_T' in tp.type.type_vars:
return tp.args[tp.type.type_vars.index('_T')]
@@ -104,16 +100,16 @@ def extract_typevar_value(tp: Instance, typevar_name: str) -> Type:
def fill_typevars(tp: Instance, type_to_fill: Instance) -> Instance:
typevar_values: typing.List[Type] = []
typevar_values: List[MypyType] = []
for typevar_arg in type_to_fill.args:
if isinstance(typevar_arg, TypeVarType):
typevar_values.append(extract_typevar_value(tp, typevar_arg.name))
return Instance(type_to_fill.type, typevar_values)
def get_argument_by_name(ctx: typing.Union[FunctionContext, MethodContext], name: str) -> Optional[Expression]:
"""Return the expression for the specific argument.
def get_call_argument_by_name(ctx: Union[FunctionContext, MethodContext], name: str) -> Optional[Expression]:
"""
Return the expression for the specific argument.
This helper should only be used with non-star arguments.
"""
if name not in ctx.callee_arg_names:
@@ -126,7 +122,7 @@ def get_argument_by_name(ctx: typing.Union[FunctionContext, MethodContext], name
return args[0]
def get_argument_type_by_name(ctx: typing.Union[FunctionContext, MethodContext], name: str) -> Optional[Type]:
def get_call_argument_type_by_name(ctx: Union[FunctionContext, MethodContext], name: str) -> Optional[MypyType]:
"""Return the type for the specific argument.
This helper should only be used with non-star arguments.
@@ -157,14 +153,38 @@ def get_setting_expr(api: 'TypeChecker', setting_name: str) -> Optional[Expressi
return None
module_file = api.modules.get(module)
for name_expr, value_expr in iter_over_assignments(module_file):
for name_expr, value_expr in iter_over_module_level_assignments(module_file):
if isinstance(name_expr, NameExpr) and name_expr.name == setting_name:
return value_expr
return None
def iter_over_assignments(class_or_module: typing.Union[ClassDef, MypyFile]
) -> typing.Iterator[typing.Tuple[Lvalue, Expression]]:
def iter_over_class_level_assignments(klass: ClassDef) -> Iterator[Tuple[str, Expression]]:
for stmt in klass.defs.body:
if not isinstance(stmt, AssignmentStmt):
continue
if len(stmt.lvalues) > 1:
# skip multiple assignments
continue
lvalue = stmt.lvalues[0]
if isinstance(lvalue, NameExpr):
yield lvalue.name, stmt.rvalue
def iter_over_module_level_assignments(module: MypyFile) -> Iterator[Tuple[str, Expression]]:
for stmt in module.defs:
if not isinstance(stmt, AssignmentStmt):
continue
if len(stmt.lvalues) > 1:
# skip multiple assignments
continue
lvalue = stmt.lvalues[0]
if isinstance(lvalue, NameExpr):
yield lvalue.name, stmt.rvalue
def iter_over_assignments_in_class(class_or_module: Union[ClassDef, MypyFile]
) -> Iterator[Tuple[str, Expression]]:
if isinstance(class_or_module, ClassDef):
statements = class_or_module.defs.body
else:
@@ -176,10 +196,12 @@ def iter_over_assignments(class_or_module: typing.Union[ClassDef, MypyFile]
if len(stmt.lvalues) > 1:
# not supported yet
continue
yield stmt.lvalues[0], stmt.rvalue
lvalue = stmt.lvalues[0]
if isinstance(lvalue, NameExpr):
yield lvalue.name, stmt.rvalue
def extract_field_setter_type(tp: Instance) -> Optional[Type]:
def extract_field_setter_type(tp: Instance) -> Optional[MypyType]:
""" Extract __set__ value of a field. """
if tp.type.has_base(fullnames.FIELD_FULLNAME):
return tp.args[0]
@@ -189,7 +211,7 @@ def extract_field_setter_type(tp: Instance) -> Optional[Type]:
return None
def extract_field_getter_type(tp: Type) -> Optional[Type]:
def extract_field_getter_type(tp: MypyType) -> Optional[MypyType]:
""" Extract return type of __get__ of subclass of Field"""
if not isinstance(tp, Instance):
return None
@@ -201,7 +223,7 @@ def extract_field_getter_type(tp: Type) -> Optional[Type]:
return None
def extract_explicit_set_type_of_model_primary_key(model: TypeInfo) -> Optional[Type]:
def extract_explicit_set_type_of_model_primary_key(model: TypeInfo) -> Optional[MypyType]:
"""
If field with primary_key=True is set on the model, extract its __set__ type.
"""
@@ -212,7 +234,7 @@ def extract_explicit_set_type_of_model_primary_key(model: TypeInfo) -> Optional[
return None
def extract_primary_key_type_for_get(model: TypeInfo) -> Optional[Type]:
def extract_primary_key_type_for_get(model: TypeInfo) -> Optional[MypyType]:
for field_name, props in metadata.get_fields_metadata(model).items():
is_primary_key = props.get('primary_key', False)
if is_primary_key:
@@ -220,11 +242,11 @@ def extract_primary_key_type_for_get(model: TypeInfo) -> Optional[Type]:
return None
def make_optional(typ: Type):
def make_optional(typ: MypyType) -> MypyType:
return UnionType.make_union([typ, NoneTyp()])
def make_required(typ: Type) -> Type:
def make_required(typ: MypyType) -> MypyType:
if not isinstance(typ, UnionType):
return typ
items = [item for item in typ.items if not isinstance(item, NoneTyp)]
@@ -232,14 +254,14 @@ def make_required(typ: Type) -> Type:
return UnionType.make_union(items)
def is_optional(typ: Type) -> bool:
def is_optional(typ: MypyType) -> bool:
if not isinstance(typ, UnionType):
return False
return any([isinstance(item, NoneTyp) for item in typ.items])
def has_any_of_bases(info: TypeInfo, bases: typing.Sequence[str]) -> bool:
def has_any_of_bases(info: TypeInfo, bases: Set[str]) -> bool:
for base_fullname in bases:
if info.has_base(base_fullname):
return True
@@ -257,10 +279,10 @@ def get_nested_meta_node_for_current_class(info: TypeInfo) -> Optional[TypeInfo]
return None
def get_assigned_value_for_class(type_info: TypeInfo, name: str) -> Optional[Expression]:
for lvalue, rvalue in iter_over_assignments(type_info.defn):
if isinstance(lvalue, NameExpr) and lvalue.name == name:
return rvalue
def get_assignment_stmt_by_name(type_info: TypeInfo, name: str) -> Optional[Expression]:
for assignment_name, call_expr in iter_over_class_level_assignments(type_info.defn):
if assignment_name == name:
return call_expr
return None
@@ -268,13 +290,13 @@ def is_field_nullable(model: TypeInfo, field_name: str) -> bool:
return metadata.get_fields_metadata(model).get(field_name, {}).get('null', False)
def is_foreign_key_like(t: Type) -> bool:
def is_foreign_key_like(t: MypyType) -> bool:
if not isinstance(t, Instance):
return False
return has_any_of_bases(t.type, (fullnames.FOREIGN_KEY_FULLNAME, fullnames.ONETOONE_FIELD_FULLNAME))
return has_any_of_bases(t.type, {fullnames.FOREIGN_KEY_FULLNAME, fullnames.ONETOONE_FIELD_FULLNAME})
def build_class_with_annotated_fields(api: 'TypeChecker', base: Type, fields: 'OrderedDict[str, Type]',
def build_class_with_annotated_fields(api: 'TypeChecker', base: MypyType, fields: 'OrderedDict[str, MypyType]',
name: str) -> Instance:
"""Build an Instance with `name` that contains the specified `fields` as attributes and extends `base`."""
# Credit: This code is largely copied/modified from TypeChecker.intersect_instance_callable and
@@ -309,7 +331,7 @@ def build_class_with_annotated_fields(api: 'TypeChecker', base: Type, fields: 'O
return Instance(info, [])
def make_named_tuple(api: 'TypeChecker', fields: 'OrderedDict[str, Type]', name: str) -> Type:
def make_named_tuple(api: 'TypeChecker', fields: 'OrderedDict[str, MypyType]', name: str) -> MypyType:
if not fields:
# No fields specified, so fallback to a subclass of NamedTuple that allows
# __getattr__ / __setattr__ for any attribute name.
@@ -317,27 +339,27 @@ def make_named_tuple(api: 'TypeChecker', fields: 'OrderedDict[str, Type]', name:
else:
fallback = build_class_with_annotated_fields(
api=api,
base=api.named_generic_type('typing.NamedTuple', []),
base=api.named_generic_type('NamedTuple', []),
fields=fields,
name=name
)
return TupleType(list(fields.values()), fallback=fallback)
def make_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, Type]',
required_keys: typing.Set[str]) -> TypedDictType:
def make_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, MypyType]',
required_keys: Set[str]) -> TypedDictType:
object_type = api.named_generic_type('mypy_extensions._TypedDict', [])
typed_dict_type = TypedDictType(fields, required_keys=required_keys, fallback=object_type)
return typed_dict_type
def make_tuple(api: 'TypeChecker', fields: typing.List[Type]) -> TupleType:
def make_tuple(api: 'TypeChecker', fields: List[MypyType]) -> TupleType:
implicit_any = AnyType(TypeOfAny.special_form)
fallback = api.named_generic_type('builtins.tuple', [implicit_any])
return TupleType(fields, fallback=fallback)
def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is_nullable: bool) -> Type:
def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is_nullable: bool) -> MypyType:
node = type_info.get(private_field_name).node
if isinstance(node, Var):
descriptor_type = node.type
@@ -347,16 +369,33 @@ def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is
return AnyType(TypeOfAny.unannotated)
def iter_over_classdefs(module_file: MypyFile) -> typing.Iterator[ClassDef]:
class IncompleteDefnException(Exception):
pass
def iter_over_toplevel_classes(module_file: MypyFile) -> Iterator[ClassDef]:
for defn in module_file.defs:
if isinstance(defn, ClassDef):
yield defn
def iter_call_assignments(klass: ClassDef) -> typing.Iterator[typing.Tuple[Lvalue, CallExpr]]:
for lvalue, rvalue in iter_over_assignments(klass):
if isinstance(rvalue, CallExpr):
yield lvalue, rvalue
def iter_call_assignments_in_class(klass: ClassDef) -> Iterator[Tuple[str, CallExpr]]:
for name, expression in iter_over_assignments_in_class(klass):
if isinstance(expression, CallExpr):
yield name, expression
def iter_over_field_inits_in_class(klass: ClassDef) -> Iterator[Tuple[str, CallExpr]]:
for lvalue, rvalue in iter_over_assignments_in_class(klass):
if isinstance(lvalue, NameExpr) and isinstance(rvalue, CallExpr):
field_name = lvalue.name
if isinstance(rvalue.callee, MemberExpr) and isinstance(rvalue.callee.node, TypeInfo):
if isinstance(rvalue.callee.node, FakeInfo):
raise IncompleteDefnException()
field_info = rvalue.callee.node
if field_info.has_base(fullnames.FIELD_FULLNAME):
yield field_name, rvalue
def get_related_manager_type_from_metadata(model_info: TypeInfo, related_manager_name: str,
@@ -394,3 +433,20 @@ def get_primary_key_field_name(model_info: TypeInfo) -> Optional[str]:
if is_primary_key:
return field_name
return None
def _get_app_models_file(app_name: str, all_modules: Dict[str, MypyFile]) -> Optional[MypyFile]:
models_module = '.'.join([app_name, 'models'])
return all_modules.get(models_module)
def get_model_info(app_name_dot_model_name: str, all_modules: Dict[str, MypyFile]) -> Optional[TypeInfo]:
""" Resolve app_name.ModelName into model fullname """
app_name, model_name = app_name_dot_model_name.split('.')
models_file = _get_app_models_file(app_name, all_modules)
if models_file is None:
return None
sym = models_file.names.get(model_name)
if sym and isinstance(sym.node, TypeInfo):
return sym.node

View File

View File

@@ -0,0 +1,21 @@
#!/usr/bin/env python
"""Django's command-line utility for administrative tasks."""
import os
import sys
def main():
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sample_django_project.settings')
try:
from django.core.management import execute_from_command_line
except ImportError as exc:
raise ImportError(
"Couldn't import Django. Are you sure it's installed and "
"available on your PYTHONPATH environment variable? Did you "
"forget to activate a virtual environment?"
) from exc
execute_from_command_line(sys.argv)
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,3 @@
from django.contrib import admin
# Register your models here.

View File

@@ -0,0 +1,5 @@
from django.apps import AppConfig
class MyappConfig(AppConfig):
label = 'myapp22'

View File

@@ -0,0 +1,6 @@
from django.db import models
# Create your models here.
class MyModel(models.Model):
pass

View File

@@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

View File

@@ -0,0 +1,3 @@
from django.shortcuts import render
# Create your views here.

View File

@@ -0,0 +1,121 @@
"""
Django settings for sample_django_project project.
Generated by 'django-admin startproject' using Django 2.2.3.
For more information on this file, see
https://docs.djangoproject.com/en/2.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.2/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'e6gj!2x(*odqwmjafrn7#35%)&rnn&^*0x-f&j0prgr--&xf+%'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'mypy_django_plugin.lib.tests.sample_django_project.myapp'
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'sample_django_project.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'sample_django_project.wsgi.application'
# Database
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
}
}
# Password validation
# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/2.2/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = '/static/'

View File

@@ -0,0 +1,21 @@
"""sample_django_project URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/2.2/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path
urlpatterns = [
path('admin/', admin.site.urls),
]

View File

@@ -0,0 +1,16 @@
"""
WSGI config for sample_django_project project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sample_django_project.settings')
application = get_wsgi_application()

View File

@@ -0,0 +1,14 @@
from mypy.options import Options
from mypy_django_plugin.lib.config import extract_app_model_aliases
from mypy_django_plugin.main import DjangoPlugin
def test_parse_django_settings():
app_model_mapping = extract_app_model_aliases('mypy_django_plugin.lib.tests.sample_django_project.root.settings')
assert app_model_mapping['myapp.MyModel'] == 'mypy_django_plugin.lib.tests.sample_django_project.myapp.models.MyModel'
def test_instantiate_plugin_with_config():
plugin = DjangoPlugin(Options())