new semanal wip 1

This commit is contained in:
Maxim Kurnikov
2019-07-16 01:22:20 +03:00
parent 9c5a6be9a7
commit b11a9a85f9
96 changed files with 4441 additions and 2370 deletions

View File

@@ -0,0 +1,264 @@
from typing import Optional, Tuple, cast
from mypy.checker import TypeChecker
from mypy.nodes import Expression, ListExpr, NameExpr, StrExpr, TupleExpr, TypeInfo
from mypy.plugin import FunctionContext
from mypy.types import AnyType, CallableType, Instance, TupleType, Type as MypyType, UnionType
from mypy_django_plugin_newsemanal.context import DjangoContext
from mypy_django_plugin_newsemanal.lib import fullnames, helpers, metadata
def extract_referred_to_type(ctx: FunctionContext, django_context: DjangoContext) -> Optional[Instance]:
api = cast(TypeChecker, ctx.api)
if 'to' not in ctx.callee_arg_names:
api.msg.fail(f'to= parameter must be set for {ctx.context.callee.fullname!r}',
context=ctx.context)
return None
arg_type = ctx.arg_types[ctx.callee_arg_names.index('to')][0]
if not isinstance(arg_type, CallableType):
to_arg_expr = ctx.args[ctx.callee_arg_names.index('to')][0]
if not isinstance(to_arg_expr, StrExpr):
# not string, not supported
return None
model_string = to_arg_expr.value
if model_string == 'self':
model_fullname = api.tscope.classes[-1].fullname()
elif '.' not in model_string:
model_fullname = api.tscope.classes[-1].module_name + '.' + model_string
else:
if django_context.app_models is not None and model_string in django_context.app_models:
model_fullname = django_context.app_models[model_string]
else:
ctx.api.fail(f'Cannot find referenced model for {model_string!r}', context=ctx.context)
return None
model_info = helpers.lookup_fully_qualified_generic(model_fullname, all_modules=api.modules)
if model_info is None or not isinstance(model_info, TypeInfo):
raise helpers.IncompleteDefnException(model_fullname)
return Instance(model_info, [])
referred_to_type = arg_type.ret_type
assert isinstance(referred_to_type, Instance)
if not referred_to_type.type.has_base(fullnames.MODEL_CLASS_FULLNAME):
ctx.api.msg.fail(f'to= parameter value must be a subclass of {fullnames.MODEL_CLASS_FULLNAME!r}',
context=ctx.context)
return None
return referred_to_type
def convert_any_to_type(typ: MypyType, replacement_type: MypyType) -> MypyType:
"""
Converts any encountered Any (in typ itself, or in generic parameters) into referred_to_type
"""
if isinstance(typ, UnionType):
converted_items = []
for item in typ.items:
converted_items.append(convert_any_to_type(item, replacement_type))
return UnionType.make_union(converted_items,
line=typ.line, column=typ.column)
if isinstance(typ, Instance):
args = []
for default_arg in typ.args:
if isinstance(default_arg, AnyType):
args.append(replacement_type)
else:
args.append(default_arg)
return helpers.reparametrize_instance(typ, args)
if isinstance(typ, AnyType):
return replacement_type
return typ
def get_referred_to_model_fullname(ctx: FunctionContext, django_context: DjangoContext) -> str:
to_arg_type = helpers.get_call_argument_type_by_name(ctx, 'to')
if isinstance(to_arg_type, CallableType):
assert isinstance(to_arg_type.ret_type, Instance)
return to_arg_type.ret_type.type.fullname()
to_arg_expr = helpers.get_call_argument_by_name(ctx, 'to')
if not isinstance(to_arg_expr, StrExpr):
raise helpers.IncompleteDefnException(f'Not a string: {to_arg_expr}')
outer_model_info = ctx.api.tscope.classes[-1]
assert isinstance(outer_model_info, TypeInfo)
model_string = to_arg_expr.value
if model_string == 'self':
return outer_model_info.fullname()
if '.' not in model_string:
# same file class
return outer_model_info.module_name + '.' + model_string
model_cls = django_context.apps_registry.get_model(model_string)
model_fullname = helpers.get_class_fullname(model_cls)
return model_fullname
def fill_descriptor_types_for_related_field(ctx: FunctionContext, django_context: DjangoContext) -> MypyType:
referred_to_fullname = get_referred_to_model_fullname(ctx, django_context)
referred_to_typeinfo = helpers.lookup_fully_qualified_generic(referred_to_fullname, ctx.api.modules)
assert isinstance(referred_to_typeinfo, TypeInfo)
referred_to_type = Instance(referred_to_typeinfo, [])
default_related_field_type = set_descriptor_types_for_field(ctx)
# replace Any with referred_to_type
args = []
for default_arg in default_related_field_type.args:
args.append(convert_any_to_type(default_arg, referred_to_type))
return helpers.reparametrize_instance(default_related_field_type, new_args=args)
def get_field_descriptor_types(field_info: TypeInfo, is_nullable: bool) -> Tuple[MypyType, MypyType]:
set_type = helpers.get_private_descriptor_type(field_info, '_pyi_private_set_type',
is_nullable=is_nullable)
get_type = helpers.get_private_descriptor_type(field_info, '_pyi_private_get_type',
is_nullable=is_nullable)
return set_type, get_type
def set_descriptor_types_for_field(ctx: FunctionContext) -> Instance:
default_return_type = cast(Instance, ctx.default_return_type)
is_nullable = helpers.parse_bool(helpers.get_call_argument_by_name(ctx, 'null'))
set_type, get_type = get_field_descriptor_types(default_return_type.type, is_nullable)
return helpers.reparametrize_instance(default_return_type, [set_type, get_type])
def transform_into_proper_return_type(ctx: FunctionContext, django_context: DjangoContext) -> MypyType:
default_return_type = ctx.default_return_type
assert isinstance(default_return_type, Instance)
if helpers.has_any_of_bases(default_return_type.type, fullnames.RELATED_FIELDS_CLASSES):
return fill_descriptor_types_for_related_field(ctx, django_context)
if default_return_type.type.has_base(fullnames.ARRAY_FIELD_FULLNAME):
return determine_type_of_array_field(ctx, django_context)
return set_descriptor_types_for_field(ctx)
def process_field_instantiation(ctx: FunctionContext, django_context: DjangoContext) -> MypyType:
# Parse __init__ parameters of field into corresponding Model's metadata
# parse_field_init_arguments_into_model_metadata(ctx)
return transform_into_proper_return_type(ctx, django_context)
def determine_type_of_array_field(ctx: FunctionContext, django_context: DjangoContext) -> MypyType:
default_return_type = set_descriptor_types_for_field(ctx)
base_field_arg_type = helpers.get_call_argument_type_by_name(ctx, 'base_field')
if not base_field_arg_type or not isinstance(base_field_arg_type, Instance):
return default_return_type
base_type = base_field_arg_type.args[1] # extract __get__ type
args = []
for default_arg in default_return_type.args:
args.append(convert_any_to_type(default_arg, base_type))
return helpers.reparametrize_instance(default_return_type, args)
# def _parse_choices_type(ctx: FunctionContext, choices_arg: Expression) -> Optional[str]:
# if isinstance(choices_arg, (TupleExpr, ListExpr)):
# # iterable of 2 element tuples of two kinds
# _, analyzed_choices = ctx.api.analyze_iterable_item_type(choices_arg)
# if isinstance(analyzed_choices, TupleType):
# first_element_type = analyzed_choices.items[0]
# if isinstance(first_element_type, Instance):
# return first_element_type.type.fullname()
# def _parse_referenced_model(ctx: FunctionContext, to_arg: Expression) -> Optional[TypeInfo]:
# if isinstance(to_arg, NameExpr) and isinstance(to_arg.node, TypeInfo):
# # reference to the model class
# return to_arg.node
#
# elif isinstance(to_arg, StrExpr):
# referenced_model_info = helpers.get_model_info(to_arg.value, ctx.api.modules)
# if referenced_model_info is not None:
# return referenced_model_info
# def parse_field_init_arguments_into_model_metadata(ctx: FunctionContext) -> None:
# outer_model = ctx.api.scope.active_class()
# if outer_model is None or not outer_model.has_base(fullnames.MODEL_CLASS_FULLNAME):
# # outside models.Model class, undetermined
# return
#
# # Determine name of the current field
# for attr_name, stmt in helpers.iter_over_class_level_assignments(outer_model.defn):
# if stmt == ctx.context:
# field_name = attr_name
# break
# else:
# return
#
# model_fields_metadata = metadata.get_fields_metadata(outer_model)
#
# # primary key
# is_primary_key = False
# primary_key_arg = helpers.get_call_argument_by_name(ctx, 'primary_key')
# if primary_key_arg:
# is_primary_key = helpers.parse_bool(primary_key_arg)
# model_fields_metadata[field_name] = {'primary_key': is_primary_key}
#
# # choices
# choices_arg = helpers.get_call_argument_by_name(ctx, 'choices')
# if choices_arg:
# choices_type_fullname = _parse_choices_type(ctx.api, choices_arg)
# if choices_type_fullname:
# model_fields_metadata[field_name]['choices_type'] = choices_type_fullname
#
# # nullability
# null_arg = helpers.get_call_argument_by_name(ctx, 'null')
# is_nullable = False
# if null_arg:
# is_nullable = helpers.parse_bool(null_arg)
# model_fields_metadata[field_name]['null'] = is_nullable
#
# # is_blankable
# blank_arg = helpers.get_call_argument_by_name(ctx, 'blank')
# is_blankable = False
# if blank_arg:
# is_blankable = helpers.parse_bool(blank_arg)
# model_fields_metadata[field_name]['blank'] = is_blankable
#
# # default
# default_arg = helpers.get_call_argument_by_name(ctx, 'default')
# if default_arg and not helpers.is_none_expr(default_arg):
# model_fields_metadata[field_name]['default_specified'] = True
#
# if helpers.has_any_of_bases(ctx.default_return_type.type, fullnames.RELATED_FIELDS_CLASSES):
# # to
# to_arg = helpers.get_call_argument_by_name(ctx, 'to')
# if to_arg:
# referenced_model = _parse_referenced_model(ctx, to_arg)
# if referenced_model is not None:
# model_fields_metadata[field_name]['to'] = referenced_model.fullname()
# else:
# model_fields_metadata[field_name]['to'] = to_arg.value
# # referenced_model = to_arg.value
# # raise helpers.IncompleteDefnException()
#
# # model_fields_metadata[field_name]['to'] = referenced_model.fullname()
# # if referenced_model is not None:
# # model_fields_metadata[field_name]['to'] = referenced_model.fullname()
# # else:
# # assert isinstance(to_arg, StrExpr)
# # model_fields_metadata[field_name]['to'] = to_arg.value
#
# # related_name
# related_name_arg = helpers.get_call_argument_by_name(ctx, 'related_name')
# if related_name_arg:
# if isinstance(related_name_arg, StrExpr):
# model_fields_metadata[field_name]['related_name'] = related_name_arg.value
# else:
# model_fields_metadata[field_name]['related_name'] = outer_model.name().lower() + '_set'

View File

@@ -0,0 +1,35 @@
from typing import cast
from mypy.checker import TypeChecker
from mypy.nodes import Argument, Var, ARG_NAMED
from mypy.plugin import FunctionContext
from mypy.types import Type as MypyType, Instance
from mypy_django_plugin_newsemanal.context import DjangoContext
from mypy_django_plugin_newsemanal.lib import helpers
def redefine_and_typecheck_model_init(ctx: FunctionContext, django_context: DjangoContext) -> MypyType:
assert isinstance(ctx.default_return_type, Instance)
api = cast(TypeChecker, ctx.api)
model_info = ctx.default_return_type.type
model_cls = django_context.get_model_class_by_fullname(model_info.fullname())
# expected_types = {}
# for field in model_cls._meta.get_fields():
# field_fullname = helpers.get_class_fullname(field.__class__)
# field_info = api.lookup_typeinfo(field_fullname)
# field_set_type = helpers.get_private_descriptor_type(field_info, '_pyi_private_set_type',
# is_nullable=False)
# field_kwarg = Argument(variable=Var(field.attname, field_set_type),
# type_annotation=field_set_type,
# initializer=None,
# kind=ARG_NAMED)
# expected_types[field.attname] = field_set_type
# for field_name, field in model_cls._meta.fields_map.items():
# print()
# print()
return ctx.default_return_type

View File

@@ -0,0 +1,255 @@
import dataclasses
from abc import ABCMeta, abstractmethod
from typing import Optional, Type, cast
from django.db.models.base import Model
from django.db.models.fields.related import ForeignKey
from mypy.newsemanal.semanal import NewSemanticAnalyzer
from mypy.nodes import ARG_NAMED_OPT, Argument, ClassDef, MDEF, SymbolTableNode, TypeInfo, Var
from mypy.plugin import ClassDefContext
from mypy.plugins import common
from mypy.types import AnyType, Instance, NoneType, Type as MypyType, UnionType
from django.contrib.postgres.fields import ArrayField
from django.db.models.fields import CharField, Field
from mypy_django_plugin_newsemanal.context import DjangoContext
from mypy_django_plugin_newsemanal.lib import helpers
from mypy_django_plugin_newsemanal.transformers import fields
from mypy_django_plugin_newsemanal.transformers.fields import get_field_descriptor_types
@dataclasses.dataclass
class ModelClassInitializer(metaclass=ABCMeta):
api: NewSemanticAnalyzer
model_classdef: ClassDef
django_context: DjangoContext
ctx: ClassDefContext
@classmethod
def from_ctx(cls, ctx: ClassDefContext, django_context: DjangoContext):
return cls(api=cast(NewSemanticAnalyzer, ctx.api),
model_classdef=ctx.cls,
django_context=django_context,
ctx=ctx)
def lookup_typeinfo_or_incomplete_defn_error(self, fullname: str) -> TypeInfo:
sym = self.api.lookup_fully_qualified_or_none(fullname)
if sym is None or not isinstance(sym.node, TypeInfo):
raise helpers.IncompleteDefnException(f'No {fullname!r} found')
return sym.node
def lookup_field_typeinfo_or_incomplete_defn_error(self, field: Field) -> TypeInfo:
fullname = helpers.get_class_fullname(field.__class__)
field_info = self.lookup_typeinfo_or_incomplete_defn_error(fullname)
return field_info
def add_new_node_to_model_class(self, name: str, typ: Instance) -> None:
# type=: type of the variable itself
var = Var(name=name, type=typ)
# var.info: type of the object variable is bound to
var.info = self.model_classdef.info
var._fullname = self.model_classdef.info.fullname() + '.' + name
var.is_initialized_in_class = True
var.is_inferred = True
self.model_classdef.info.names[name] = SymbolTableNode(MDEF, var, plugin_generated=True)
# assert self.model_classdef.info == self.api.type
# self.api.add_symbol_table_node(name, SymbolTableNode(MDEF, var, plugin_generated=True))
def convert_any_to_type(self, typ: MypyType, referred_to_type: MypyType) -> MypyType:
if isinstance(typ, UnionType):
converted_items = []
for item in typ.items:
converted_items.append(self.convert_any_to_type(item, referred_to_type))
return UnionType.make_union(converted_items,
line=typ.line, column=typ.column)
if isinstance(typ, Instance):
args = []
for default_arg in typ.args:
if isinstance(default_arg, AnyType):
args.append(referred_to_type)
else:
args.append(default_arg)
return helpers.reparametrize_instance(typ, args)
if isinstance(typ, AnyType):
return referred_to_type
return typ
def get_field_set_type(self, field: Field, method: str) -> MypyType:
target_field = field
if isinstance(field, ForeignKey):
target_field = field.target_field
field_fullname = helpers.get_class_fullname(target_field.__class__)
field_info = self.lookup_typeinfo_or_incomplete_defn_error(field_fullname)
field_set_type = helpers.get_private_descriptor_type(field_info, '_pyi_private_set_type',
is_nullable=self.get_field_nullability(field, method))
if isinstance(target_field, ArrayField):
argument_field_type = self.get_field_set_type(target_field.base_field, method)
field_set_type = self.convert_any_to_type(field_set_type, argument_field_type)
return field_set_type
def get_field_nullability(self, field: Field, method: Optional[str]) -> bool:
nullable = field.null
if not nullable and isinstance(field, CharField) and field.blank:
return True
if method == '__init__':
if field.primary_key or isinstance(field, ForeignKey):
return True
return nullable
def get_field_kind(self, field: Field, method: str):
if method == '__init__':
# all arguments are optional in __init__
return ARG_NAMED_OPT
def get_primary_key_field(self, model_cls: Type[Model]) -> Field:
for field in model_cls._meta.get_fields():
if isinstance(field, Field):
if field.primary_key:
return field
raise ValueError('No primary key defined')
def make_field_kwarg(self, name: str, field: Field, method: str) -> Argument:
field_set_type = self.get_field_set_type(field, method)
kind = self.get_field_kind(field, method)
field_kwarg = Argument(variable=Var(name, field_set_type),
type_annotation=field_set_type,
initializer=None,
kind=kind)
return field_kwarg
def get_field_kwargs(self, model_cls: Type[Model], method: str):
field_kwargs = []
if method == '__init__':
# add primary key `pk`
primary_key_field = self.get_primary_key_field(model_cls)
field_kwarg = self.make_field_kwarg('pk', primary_key_field, method)
field_kwargs.append(field_kwarg)
for field in model_cls._meta.get_fields():
if isinstance(field, Field):
field_kwarg = self.make_field_kwarg(field.attname, field, method)
field_kwargs.append(field_kwarg)
if isinstance(field, ForeignKey):
attname = field.name
related_model_fullname = helpers.get_class_fullname(field.related_model)
model_info = self.lookup_typeinfo_or_incomplete_defn_error(related_model_fullname)
is_nullable = self.get_field_nullability(field, method)
field_set_type = Instance(model_info, [])
if is_nullable:
field_set_type = helpers.make_optional(field_set_type)
kind = self.get_field_kind(field, method)
field_kwarg = Argument(variable=Var(attname, field_set_type),
type_annotation=field_set_type,
initializer=None,
kind=kind)
field_kwargs.append(field_kwarg)
return field_kwargs
@abstractmethod
def run(self) -> None:
raise NotImplementedError()
class InjectAnyAsBaseForNestedMeta(ModelClassInitializer):
"""
Replaces
class MyModel(models.Model):
class Meta:
pass
with
class MyModel(models.Model):
class Meta(Any):
pass
to get around incompatible Meta inner classes for different models.
"""
def run(self) -> None:
meta_node = helpers.get_nested_meta_node_for_current_class(self.model_classdef.info)
if meta_node is None:
return None
meta_node.fallback_to_any = True
class AddDefaultPrimaryKey(ModelClassInitializer):
def run(self) -> None:
model_cls = self.django_context.get_model_class_by_fullname(self.model_classdef.fullname)
if model_cls is None:
return
auto_field = model_cls._meta.auto_field
if auto_field and not self.model_classdef.info.has_readable_member(auto_field.attname):
# autogenerated field
auto_field_fullname = helpers.get_class_fullname(auto_field.__class__)
auto_field_info = self.lookup_typeinfo_or_incomplete_defn_error(auto_field_fullname)
set_type, get_type = fields.get_field_descriptor_types(auto_field_info, is_nullable=False)
self.add_new_node_to_model_class(auto_field.attname, Instance(auto_field_info,
[set_type, get_type]))
class AddRelatedModelsId(ModelClassInitializer):
def run(self) -> None:
model_cls = self.django_context.get_model_class_by_fullname(self.model_classdef.fullname)
if model_cls is None:
return
for field in model_cls._meta.get_fields():
if isinstance(field, ForeignKey):
rel_primary_key_field = self.get_primary_key_field(field.related_model)
field_info = self.lookup_field_typeinfo_or_incomplete_defn_error(rel_primary_key_field)
is_nullable = self.get_field_nullability(field, None)
set_type, get_type = get_field_descriptor_types(field_info, is_nullable)
self.add_new_node_to_model_class(field.attname,
Instance(field_info, [set_type, get_type]))
class AddManagers(ModelClassInitializer):
def run(self):
model_cls = self.django_context.get_model_class_by_fullname(self.model_classdef.fullname)
if model_cls is None:
return
for manager_name, manager in model_cls._meta.managers_map.items():
if manager_name not in self.model_classdef.info.names:
manager_fullname = helpers.get_class_fullname(manager.__class__)
manager_info = self.lookup_typeinfo_or_incomplete_defn_error(manager_fullname)
manager = Instance(manager_info, [Instance(self.model_classdef.info, [])])
self.add_new_node_to_model_class(manager_name, manager)
# add _default_manager
if '_default_manager' not in self.model_classdef.info.names:
default_manager_fullname = helpers.get_class_fullname(model_cls._meta.default_manager.__class__)
default_manager_info = self.lookup_typeinfo_or_incomplete_defn_error(default_manager_fullname)
default_manager = Instance(default_manager_info, [Instance(self.model_classdef.info, [])])
self.add_new_node_to_model_class('_default_manager', default_manager)
class AddInitMethod(ModelClassInitializer):
def run(self):
model_cls = self.django_context.get_model_class_by_fullname(self.model_classdef.info.fullname())
if model_cls is None:
return
field_kwargs = self.get_field_kwargs(model_cls, '__init__')
common.add_method(self.ctx, '__init__', field_kwargs, NoneType())
def process_model_class(ctx: ClassDefContext,
django_context: DjangoContext) -> None:
initializers = [
InjectAnyAsBaseForNestedMeta,
AddDefaultPrimaryKey,
AddRelatedModelsId,
AddManagers,
AddInitMethod
]
for initializer_cls in initializers:
try:
initializer_cls.from_ctx(ctx, django_context).run()
except helpers.IncompleteDefnException:
if not ctx.api.final_iteration:
ctx.api.defer()

View File

@@ -0,0 +1,39 @@
from mypy.plugin import AnalyzeTypeContext, FunctionContext
from mypy.types import AnyType, Instance, Type as MypyType, TypeOfAny
from mypy_django_plugin_newsemanal.lib import fullnames, helpers
def set_first_generic_param_as_default_for_second(ctx: AnalyzeTypeContext, fullname: str) -> MypyType:
if not ctx.type.args:
try:
return ctx.api.named_type(fullname, [AnyType(TypeOfAny.explicit),
AnyType(TypeOfAny.explicit)])
except KeyError:
# really should never happen
return AnyType(TypeOfAny.explicit)
args = ctx.type.args
if len(args) == 1:
args = [args[0], args[0]]
analyzed_args = [ctx.api.analyze_type(arg) for arg in args]
ctx.api.analyze_type(ctx.type)
try:
return ctx.api.named_type(fullname, analyzed_args)
except KeyError:
return AnyType(TypeOfAny.explicit)
def determine_proper_manager_type(ctx: FunctionContext) -> MypyType:
ret = ctx.default_return_type
assert isinstance(ret, Instance)
if not ctx.api.tscope.classes:
# not in class
return ret
outer_model_info = ctx.api.tscope.classes[0]
if not outer_model_info.has_base(fullnames.MODEL_CLASS_FULLNAME):
return ret
return helpers.reparametrize_instance(ret, [Instance(outer_model_info, [])])

View File

@@ -0,0 +1,18 @@
from mypy.nodes import TypeInfo
from mypy.plugin import FunctionContext
from mypy.types import Type as MypyType, TypeType, Instance
from mypy_django_plugin_newsemanal.context import DjangoContext
from mypy_django_plugin_newsemanal.lib import helpers
def get_user_model_hook(ctx: FunctionContext, django_context: DjangoContext) -> MypyType:
auth_user_model = django_context.settings.AUTH_USER_MODEL
model_cls = django_context.apps_registry.get_model(auth_user_model)
model_cls_fullname = helpers.get_class_fullname(model_cls)
model_info = helpers.lookup_fully_qualified_generic(model_cls_fullname, ctx.api.modules)
assert isinstance(model_info, TypeInfo)
return TypeType(Instance(model_info, []))