cleanups, fix settings

This commit is contained in:
Maxim Kurnikov
2018-12-01 16:26:53 +03:00
parent 60b1c48ade
commit fcd659837e
26 changed files with 316 additions and 1173 deletions

View File

@@ -1,9 +1,9 @@
import typing
from typing import Dict, Optional, NamedTuple
from mypy.nodes import SymbolTableNode, Var, Expression, StrExpr, MypyFile, TypeInfo
from mypy.nodes import Expression, StrExpr, MypyFile, TypeInfo
from mypy.plugin import FunctionContext
from mypy.types import Type, Instance, UnionType, NoneTyp
from mypy.types import Type, UnionType, NoneTyp
MODEL_CLASS_FULLNAME = 'django.db.models.base.Model'
QUERYSET_CLASS_FULLNAME = 'django.db.models.query.QuerySet'
@@ -11,14 +11,6 @@ FOREIGN_KEY_FULLNAME = 'django.db.models.fields.related.ForeignKey'
ONETOONE_FIELD_FULLNAME = 'django.db.models.fields.related.OneToOneField'
DUMMY_SETTINGS_BASE_CLASS = 'django.conf._DjangoConfLazyObject'
def create_new_symtable_node(name: str, kind: int, instance: Instance) -> SymbolTableNode:
new_var = Var(name, instance)
new_var.info = instance.type
return SymbolTableNode(kind, new_var,
plugin_generated=True)
Argument = NamedTuple('Argument', fields=[
('arg', Expression),
('arg_type', Type)

View File

@@ -1,81 +1,24 @@
import os
from typing import Callable, Optional, cast
from typing import Callable, Optional
from mypy.nodes import AssignmentStmt, CallExpr, MemberExpr, StrExpr, NameExpr
from mypy.options import Options
from mypy.plugin import Plugin, FunctionContext, ClassDefContext
from mypy.semanal import SemanticAnalyzerPass2
from mypy.types import Type, Instance
from mypy.plugin import Plugin, FunctionContext, ClassDefContext, AnalyzeTypeContext
from mypy.types import Type
from mypy_django_plugin import helpers, monkeypatch
from mypy_django_plugin.plugins.meta_inner_class import inject_any_as_base_for_nested_class_meta
from mypy_django_plugin.plugins.objects_queryset import set_objects_queryset_to_model_class
from mypy_django_plugin.plugins.fields import determine_type_of_array_field, \
add_int_id_attribute_if_primary_key_true_is_not_present
from mypy_django_plugin.plugins.related_fields import set_fieldname_attrs_for_related_fields, add_new_var_node_to_class, \
extract_to_parameter_as_get_ret_type
from mypy_django_plugin.plugins.setup_settings import DjangoConfSettingsInitializerHook
from mypy_django_plugin.plugins.fields import determine_type_of_array_field
from mypy_django_plugin.plugins.models import process_model_class
from mypy_django_plugin.plugins.related_fields import extract_to_parameter_as_get_ret_type_for_related_field
from mypy_django_plugin.plugins.settings import DjangoConfSettingsInitializerHook
base_model_classes = {helpers.MODEL_CLASS_FULLNAME}
def add_related_managers_from_referred_foreign_keys_to_model(ctx: ClassDefContext) -> None:
api = cast(SemanticAnalyzerPass2, ctx.api)
for stmt in ctx.cls.defs.body:
if not isinstance(stmt, AssignmentStmt):
continue
if len(stmt.lvalues) > 1:
# not supported yet
continue
rvalue = stmt.rvalue
if not isinstance(rvalue, CallExpr):
continue
if (not isinstance(rvalue.callee, MemberExpr)
or not rvalue.callee.fullname in {helpers.FOREIGN_KEY_FULLNAME,
helpers.ONETOONE_FIELD_FULLNAME}):
continue
if 'related_name' not in rvalue.arg_names:
# positional related_name is not supported yet
continue
related_name = rvalue.args[rvalue.arg_names.index('related_name')].value
if 'to' in rvalue.arg_names:
expr = rvalue.args[rvalue.arg_names.index('to')]
else:
# first positional argument
expr = rvalue.args[0]
if isinstance(expr, StrExpr):
model_typeinfo = helpers.get_model_type_from_string(expr,
all_modules=api.modules)
if model_typeinfo is None:
continue
elif isinstance(expr, NameExpr):
model_typeinfo = expr.node
else:
continue
if rvalue.callee.fullname == helpers.FOREIGN_KEY_FULLNAME:
typ = api.named_type_or_none(helpers.QUERYSET_CLASS_FULLNAME,
args=[Instance(ctx.cls.info, [])])
else:
typ = Instance(ctx.cls.info, [])
if typ is None:
continue
add_new_var_node_to_class(model_typeinfo, related_name, typ)
class TransformModelClassHook(object):
def __call__(self, ctx: ClassDefContext) -> None:
base_model_classes.add(ctx.cls.fullname)
set_fieldname_attrs_for_related_fields(ctx)
set_objects_queryset_to_model_class(ctx)
inject_any_as_base_for_nested_class_meta(ctx)
add_related_managers_from_referred_foreign_keys_to_model(ctx)
add_int_id_attribute_if_primary_key_true_is_not_present(ctx)
process_model_class(ctx)
class DjangoPlugin(Plugin):
@@ -89,7 +32,6 @@ class DjangoPlugin(Plugin):
if self.django_settings:
monkeypatch.load_graph_to_add_settings_file_as_a_source_seed(self.django_settings)
monkeypatch.inject_dependencies(self.django_settings)
# monkeypatch.process_settings_before_dependants(self.django_settings)
else:
monkeypatch.restore_original_load_graph()
monkeypatch.restore_original_dependencies_handling()
@@ -98,7 +40,7 @@ class DjangoPlugin(Plugin):
) -> Optional[Callable[[FunctionContext], Type]]:
if fullname in {helpers.FOREIGN_KEY_FULLNAME,
helpers.ONETOONE_FIELD_FULLNAME}:
return extract_to_parameter_as_get_ret_type
return extract_to_parameter_as_get_ret_type_for_related_field
# if fullname == helpers.ONETOONE_FIELD_FULLNAME:
# return OneToOneFieldHook(settings=self.django_settings)

View File

@@ -1,7 +1,7 @@
from typing import Optional, List, Sequence, NamedTuple, Tuple
from mypy import checkexpr
from mypy.argmap import map_actuals_to_formals
from mypy.checkexpr import map_actuals_to_formals
from mypy.checkmember import analyze_member_access
from mypy.expandtype import freshen_function_type_vars
from mypy.messages import MessageBuilder
@@ -68,7 +68,6 @@ class PatchedExpressionChecker(checkexpr.ExpressionChecker):
on which the method is being called
"""
arg_messages = arg_messages or self.msg
if isinstance(callee, CallableType):
if callable_name is None and callee.name:
callable_name = callee.name

View File

@@ -1,11 +1,5 @@
from typing import Iterator, List, cast
from mypy.nodes import ClassDef, AssignmentStmt, CallExpr
from mypy.plugin import FunctionContext, ClassDefContext
from mypy.semanal import SemanticAnalyzerPass2
from mypy.types import Type, Instance
from mypy_django_plugin.plugins.related_fields import add_new_var_node_to_class
from mypy.plugin import FunctionContext
from mypy.types import Type
def determine_type_of_array_field(ctx: FunctionContext) -> Type:
@@ -15,27 +9,3 @@ def determine_type_of_array_field(ctx: FunctionContext) -> Type:
base_field_arg_type = ctx.arg_types[ctx.arg_names.index('base_field')][0]
return ctx.api.named_generic_type(ctx.context.callee.fullname,
args=[base_field_arg_type.type.names['__get__'].type.ret_type])
def get_assignments(klass: ClassDef) -> List[AssignmentStmt]:
stmts = []
for stmt in klass.defs.body:
if not isinstance(stmt, AssignmentStmt):
continue
if len(stmt.lvalues) > 1:
# not supported yet
continue
stmts.append(stmt)
return stmts
def add_int_id_attribute_if_primary_key_true_is_not_present(ctx: ClassDefContext) -> None:
api = cast(SemanticAnalyzerPass2, ctx.api)
for stmt in get_assignments(ctx.cls):
if (isinstance(stmt.rvalue, CallExpr)
and 'primary_key' in stmt.rvalue.arg_names
and api.parse_bool(stmt.rvalue.args[stmt.rvalue.arg_names.index('primary_key')])):
break
else:
add_new_var_node_to_class(ctx.cls.info, 'id', api.builtin_type('builtins.int'))

View File

@@ -1,12 +0,0 @@
from mypy.nodes import TypeInfo
from mypy.plugin import ClassDefContext
def inject_any_as_base_for_nested_class_meta(ctx: ClassDefContext) -> None:
if 'Meta' not in ctx.cls.info.names:
return None
sym = ctx.cls.info.names['Meta']
if not isinstance(sym.node, TypeInfo):
return None
sym.node.fallback_to_any = True

View File

@@ -0,0 +1,180 @@
from typing import cast, Iterator, Tuple, Optional
from mypy.nodes import ClassDef, AssignmentStmt, CallExpr, MemberExpr, StrExpr, NameExpr, MDEF, TypeInfo, Var, SymbolTableNode, \
Lvalue, Expression, Statement
from mypy.plugin import ClassDefContext
from mypy.semanal import SemanticAnalyzerPass2
from mypy.types import Instance
from mypy_django_plugin import helpers
def add_new_var_node_to_class(class_type: TypeInfo, name: str, typ: Instance) -> None:
var = Var(name=name, type=typ)
var.info = typ.type
var._fullname = class_type.fullname() + '.' + name
var.is_inferred = True
var.is_initialized_in_class = True
class_type.names[name] = SymbolTableNode(MDEF, var)
def iter_over_assignments(klass: ClassDef) -> Iterator[Tuple[Lvalue, Expression]]:
for stmt in klass.defs.body:
if not isinstance(stmt, AssignmentStmt):
continue
if len(stmt.lvalues) > 1:
# not supported yet
continue
yield stmt.lvalues[0], stmt.rvalue
def iter_call_assignments(klass: ClassDef) -> Iterator[Tuple[Lvalue, CallExpr]]:
for lvalue, rvalue in iter_over_assignments(klass):
if not isinstance(rvalue, CallExpr):
continue
yield lvalue, rvalue
def iter_over_one_to_n_related_fields(klass: ClassDef, api: SemanticAnalyzerPass2) -> Iterator[Tuple[NameExpr, CallExpr]]:
for lvalue, rvalue in iter_call_assignments(klass):
if (isinstance(lvalue, NameExpr)
and isinstance(rvalue.callee, MemberExpr)):
if rvalue.callee.fullname in {helpers.FOREIGN_KEY_FULLNAME,
helpers.ONETOONE_FIELD_FULLNAME}:
yield lvalue, rvalue
def get_nested_meta_class(model_type: TypeInfo) -> Optional[TypeInfo]:
metaclass_sym = model_type.names.get('Meta')
if metaclass_sym is not None and isinstance(metaclass_sym.node, TypeInfo):
return metaclass_sym.node
return None
def is_abstract_model(ctx: ClassDefContext) -> bool:
meta_node = get_nested_meta_class(ctx.cls.info)
if meta_node is None:
return False
for lvalue, rvalue in iter_over_assignments(meta_node.defn):
if isinstance(lvalue, NameExpr) and lvalue.name == 'abstract':
is_abstract = ctx.api.parse_bool(rvalue)
if is_abstract:
# abstract model do not need 'objects' queryset
return True
return False
def set_fieldname_attrs_for_related_fields(ctx: ClassDefContext) -> None:
api = ctx.api
for lvalue, rvalue in iter_over_one_to_n_related_fields(ctx.cls, api):
property_name = lvalue.name + '_id'
add_new_var_node_to_class(ctx.cls.info, property_name,
typ=api.named_type('__builtins__.int'))
def add_int_id_attribute_if_primary_key_true_is_not_present(ctx: ClassDefContext) -> None:
api = cast(SemanticAnalyzerPass2, ctx.api)
if is_abstract_model(ctx):
return None
for _, rvalue in iter_call_assignments(ctx.cls):
if ('primary_key' in rvalue.arg_names and
api.parse_bool(rvalue.args[rvalue.arg_names.index('primary_key')])):
break
else:
add_new_var_node_to_class(ctx.cls.info, 'id', api.builtin_type('builtins.int'))
def set_objects_queryset_to_model_class(ctx: ClassDefContext) -> None:
# search over mro
objects_sym = ctx.cls.info.get('objects')
if objects_sym is not None:
return None
# only direct Meta class
if is_abstract_model(ctx):
# abstract model do not need 'objects' queryset
return None
api = cast(SemanticAnalyzerPass2, ctx.api)
typ = api.named_type_or_none(helpers.QUERYSET_CLASS_FULLNAME,
args=[Instance(ctx.cls.info, [])])
if not typ:
return None
add_new_var_node_to_class(ctx.cls.info, 'objects', typ=typ)
def inject_any_as_base_for_nested_class_meta(ctx: ClassDefContext) -> None:
meta_node = get_nested_meta_class(ctx.cls.info)
if meta_node is None:
return None
meta_node.fallback_to_any = True
def is_model_defn(defn: Statement, api: SemanticAnalyzerPass2) -> bool:
if not isinstance(defn, ClassDef):
return False
for base_type_expr in defn.base_type_exprs:
# api.accept(base_type_expr)
fullname = getattr(base_type_expr, 'fullname', None)
if fullname == helpers.MODEL_CLASS_FULLNAME:
return True
return False
def iter_over_models(ctx: ClassDefContext) -> Iterator[ClassDef]:
for module_name, module_file in ctx.api.modules.items():
for defn in module_file.defs:
if is_model_defn(defn, api=cast(SemanticAnalyzerPass2, ctx.api)):
yield defn
def extract_to_value_or_none(field_expr: CallExpr, ctx: ClassDefContext) -> Optional[TypeInfo]:
if 'to' in field_expr.arg_names:
ref_expr = field_expr.args[field_expr.arg_names.index('to')]
else:
# first positional argument
ref_expr = field_expr.args[0]
if isinstance(ref_expr, StrExpr):
model_typeinfo = helpers.get_model_type_from_string(ref_expr,
all_modules=ctx.api.modules)
return model_typeinfo
elif isinstance(ref_expr, NameExpr):
return ref_expr.node
def get_related_field_type(rvalue: CallExpr, api: SemanticAnalyzerPass2,
related_model_typ: TypeInfo) -> Optional[Instance]:
if rvalue.callee.fullname == helpers.FOREIGN_KEY_FULLNAME:
return api.named_type_or_none(helpers.QUERYSET_CLASS_FULLNAME,
args=[Instance(related_model_typ, [])])
else:
return Instance(related_model_typ, [])
def add_related_managers(ctx: ClassDefContext) -> None:
for model_defn in iter_over_models(ctx):
for _, rvalue in iter_over_one_to_n_related_fields(model_defn, ctx.api):
if 'related_name' not in rvalue.arg_names:
# positional related_name is not supported yet
return
related_name = rvalue.args[rvalue.arg_names.index('related_name')].value
ref_to_typ = extract_to_value_or_none(rvalue, ctx)
if ref_to_typ is not None:
if ref_to_typ.fullname() == ctx.cls.info.fullname():
typ = get_related_field_type(rvalue, ctx.api,
related_model_typ=model_defn.info)
if typ is None:
return
add_new_var_node_to_class(ctx.cls.info, related_name, typ)
def process_model_class(ctx: ClassDefContext) -> None:
# add_related_managers(ctx)
inject_any_as_base_for_nested_class_meta(ctx)
set_fieldname_attrs_for_related_fields(ctx)
add_int_id_attribute_if_primary_key_true_is_not_present(ctx)
set_objects_queryset_to_model_class(ctx)

View File

@@ -1,36 +0,0 @@
from typing import cast
from mypy.nodes import MDEF, AssignmentStmt
from mypy.plugin import ClassDefContext
from mypy.semanal import SemanticAnalyzerPass2
from mypy.types import Instance
from mypy_django_plugin import helpers
def set_objects_queryset_to_model_class(ctx: ClassDefContext) -> None:
# search over mro
objects_sym = ctx.cls.info.get('objects')
if objects_sym is not None:
return None
# only direct Meta class
metaclass_sym = ctx.cls.info.names.get('Meta')
# skip if abstract
if metaclass_sym is not None:
for stmt in metaclass_sym.node.defn.defs.body:
if (isinstance(stmt, AssignmentStmt) and len(stmt.lvalues) == 1
and stmt.lvalues[0].name == 'abstract'):
is_abstract = ctx.api.parse_bool(stmt.rvalue)
if is_abstract:
return None
api = cast(SemanticAnalyzerPass2, ctx.api)
typ = api.named_type_or_none(helpers.QUERYSET_CLASS_FULLNAME,
args=[Instance(ctx.cls.info, [])])
if not typ:
return None
ctx.cls.info.names['objects'] = helpers.create_new_symtable_node('objects',
kind=MDEF,
instance=typ)

View File

@@ -1,18 +1,12 @@
import typing
from typing import Optional, cast
from django.conf import Settings
from mypy.checker import TypeChecker
from mypy.nodes import MDEF, AssignmentStmt, MypyFile, StrExpr, TypeInfo, NameExpr, Var, SymbolTableNode
from mypy.plugin import FunctionContext, ClassDefContext
from mypy.nodes import StrExpr
from mypy.plugin import FunctionContext
from mypy.types import Type, CallableType, Instance, AnyType, TypeOfAny
from mypy_django_plugin import helpers
from mypy_django_plugin.helpers import get_models_file
def extract_related_name_value(ctx: FunctionContext) -> str:
return ctx.args[ctx.arg_names.index('related_name')][0].value
def reparametrize_with(instance: Instance, new_typevars: typing.List[Type]):
@@ -55,44 +49,9 @@ def get_valid_to_value_or_none(ctx: FunctionContext) -> Optional[Instance]:
return referred_to_type
def add_new_var_node_to_class(class_type: TypeInfo, name: str, typ: Instance) -> None:
var = Var(name=name, type=typ)
var.info = typ.type
var._fullname = class_type.fullname() + '.' + name
var.is_inferred = True
var.is_initialized_in_class = True
class_type.names[name] = SymbolTableNode(MDEF, var)
def extract_to_parameter_as_get_ret_type(ctx: FunctionContext) -> Type:
def extract_to_parameter_as_get_ret_type_for_related_field(ctx: FunctionContext) -> Type:
referred_to_type = get_valid_to_value_or_none(ctx)
if referred_to_type is None:
# couldn't extract to= value
return fill_typevars_with_any(ctx.default_return_type)
return reparametrize_with(ctx.default_return_type, [referred_to_type])
def set_fieldname_attrs_for_related_fields(ctx: ClassDefContext) -> None:
api = ctx.api
for stmt in ctx.cls.defs.body:
if not isinstance(stmt, AssignmentStmt):
continue
if not hasattr(stmt.rvalue, 'callee'):
continue
if len(stmt.lvalues) > 1:
# multiple lvalues not supported for now
continue
expr = stmt.lvalues[0]
if not isinstance(expr, NameExpr):
continue
name = expr.name
rvalue_callee = stmt.rvalue.callee
if rvalue_callee.fullname in {helpers.FOREIGN_KEY_FULLNAME,
helpers.ONETOONE_FIELD_FULLNAME}:
name += '_id'
new_node = helpers.create_new_symtable_node(name,
kind=MDEF,
instance=api.named_type('__builtins__.int'))
ctx.cls.info.names[name] = new_node

View File

@@ -0,0 +1,61 @@
from typing import cast, List
from mypy.nodes import Var, Context, SymbolNode, SymbolTableNode
from mypy.plugin import ClassDefContext
from mypy.semanal import SemanticAnalyzerPass2
from mypy.types import Instance, UnionType, NoneTyp, Type
from mypy_django_plugin import helpers
def get_error_context(node: SymbolNode) -> Context:
context = Context()
context.set_line(node)
return context
def filter_out_nones(typ: UnionType) -> List[Type]:
return [item for item in typ.items if not isinstance(item, NoneTyp)]
def copy_sym_of_instance(sym: SymbolTableNode) -> SymbolTableNode:
copied = sym.copy()
copied.node.info = sym.type.type
return copied
def add_settings_to_django_conf_object(ctx: ClassDefContext,
settings_module: str) -> None:
api = cast(SemanticAnalyzerPass2, ctx.api)
if settings_module not in api.modules:
return None
settings_file = api.modules[settings_module]
for name, sym in settings_file.names.items():
if name.isupper() and isinstance(sym.node, Var):
if isinstance(sym.type, Instance):
copied = sym.copy()
copied.node.info = sym.type.type
ctx.cls.info.names[name] = copied
elif isinstance(sym.type, UnionType):
instances = filter_out_nones(sym.type)
if len(instances) > 1:
# plain unions not supported yet
continue
typ = instances[0]
if isinstance(typ, Instance):
copied = sym.copy()
copied.node.info = typ.type
ctx.cls.info.names[name] = copied
class DjangoConfSettingsInitializerHook(object):
def __init__(self, settings_module: str):
self.settings_module = settings_module
def __call__(self, ctx: ClassDefContext) -> None:
if not self.settings_module:
return
add_settings_to_django_conf_object(ctx, self.settings_module)

View File

@@ -1,42 +0,0 @@
from typing import Optional, Any, cast
from mypy.nodes import Var, Context, GDEF
from mypy.options import Options
from mypy.plugin import ClassDefContext
from mypy.semanal import SemanticAnalyzerPass2
from mypy.types import Instance
def add_settings_to_django_conf_object(ctx: ClassDefContext,
settings_module: str) -> Optional[Any]:
api = cast(SemanticAnalyzerPass2, ctx.api)
if settings_module not in api.modules:
return None
settings_file = api.modules[settings_module]
for name, sym in settings_file.names.items():
if name.isupper():
if not isinstance(sym.node, Var) or not isinstance(sym.type, Instance):
error_context = Context()
error_context.set_line(sym.node)
api.msg.fail("Need type annotation for '{}'".format(sym.node.name()),
context=error_context,
file=settings_file.path,
origin=Context())
continue
sym_copy = sym.copy()
sym_copy.node.info = sym_copy.type.type
sym_copy.kind = GDEF
ctx.cls.info.names[name] = sym_copy
class DjangoConfSettingsInitializerHook(object):
def __init__(self, settings_module: str):
self.settings_module = settings_module
def __call__(self, ctx: ClassDefContext) -> None:
if not self.settings_module:
return
add_settings_to_django_conf_object(ctx, self.settings_module)