finish strict_optional support, enable it for typechecking of django tests

2025-12-07 04:34:29 +08:00 · 2019-02-18 00:52:56 +03:00
parent 400a0f0486
commit f980311be0
16 changed files with 140 additions and 41 deletions
--- a/django-stubs/apps/config.pyi
+++ b/django-stubs/apps/config.pyi
@@ -12,7 +12,7 @@ class AppConfig:
    verbose_name: str = ...
    path: str = ...
    models_module: None = ...
-    models: Optional[Dict[str, Type[Model]]] = ...
+    models: Dict[str, Type[Model]] = ...
    def __init__(self, app_name: str, app_module: Optional[Any]) -> None: ...
    @classmethod
    def create(cls, entry: str) -> AppConfig: ...
--- a/django-stubs/contrib/admin/options.pyi
+++ b/django-stubs/contrib/admin/options.pyi
@@ -57,9 +57,15 @@ class BaseModelAdmin:
    checks_class: Any = ...
    def check(self, **kwargs: Any) -> List[Union[str, Error]]: ...
    def __init__(self) -> None: ...
-    def formfield_for_dbfield(self, db_field: Field, request: WSGIRequest, **kwargs: Any) -> Optional[Field]: ...
-    def formfield_for_choice_field(self, db_field: Field, request: WSGIRequest, **kwargs: Any) -> TypedChoiceField: ...
-    def get_field_queryset(self, db: None, db_field: RelatedField, request: WSGIRequest) -> Optional[QuerySet]: ...
+    def formfield_for_dbfield(
+        self, db_field: Field, request: Optional[WSGIRequest], **kwargs: Any
+    ) -> Optional[Field]: ...
+    def formfield_for_choice_field(
+        self, db_field: Field, request: Optional[WSGIRequest], **kwargs: Any
+    ) -> TypedChoiceField: ...
+    def get_field_queryset(
+        self, db: None, db_field: RelatedField, request: Optional[WSGIRequest]
+    ) -> Optional[QuerySet]: ...
    def formfield_for_foreignkey(
        self, db_field: ForeignKey, request: Optional[WSGIRequest], **kwargs: Any
    ) -> Optional[ModelChoiceField]: ...
@@ -90,7 +96,7 @@ class BaseModelAdmin:
 class ModelAdmin(BaseModelAdmin):
    formfield_overrides: Any
    list_display: Sequence[Union[str, Callable]] = ...
-    list_display_links: Sequence[Union[str, Callable]] = ...
+    list_display_links: Optional[Sequence[Union[str, Callable]]] = ...
    list_filter: Sequence[Union[str, Type[ListFilter], Tuple[str, Type[ListFilter]]]] = ...
    list_select_related: Union[bool, Sequence[str]] = ...
    list_per_page: int = ...
--- a/django-stubs/db/migrations/state.pyi
+++ b/django-stubs/db/migrations/state.pyi
@@ -22,9 +22,9 @@ class ModelState:
    name: str
    app_label: str
    fields: List[Tuple[str, Field]]
-    options: Optional[Dict[str, Any]] = ...
-    bases: Optional[Tuple[Type[Model]]] = ...
-    managers: Optional[List[Tuple[str, Manager]]] = ...
+    options: Dict[str, Any] = ...
+    bases: Tuple[Type[Model]] = ...
+    managers: List[Tuple[str, Manager]] = ...
    def __init__(
        self,
        app_label: str,
--- a/django-stubs/db/models/fields/init.pyi
+++ b/django-stubs/db/models/fields/init.pyi
@@ -224,7 +224,7 @@ class GenericIPAddressField(Field[_ST, _GT]):
 class DateTimeCheckMixin: ...

 class DateField(DateTimeCheckMixin, Field[_ST, _GT]):
-    _pyi_private_set_type: Union[str, date, datetime, Combinable]
+    _pyi_private_set_type: Union[str, date, Combinable]
    _pyi_private_get_type: date
    def __init__(
        self,
--- a/django-stubs/db/models/manager.pyi
+++ b/django-stubs/db/models/manager.pyi
@@ -4,7 +4,6 @@ from django.db.models.base import Model
 from django.db.models.query import QuerySet

 _T = TypeVar("_T", bound=Model, covariant=True)
-_Self = TypeVar("_Self", bound="BaseManager")

 class BaseManager(QuerySet[_T]):
    creation_counter: int = ...
@@ -17,9 +16,7 @@ class BaseManager(QuerySet[_T]):
    def deconstruct(self) -> Tuple[bool, str, None, Tuple, Dict[str, int]]: ...
    def check(self, **kwargs: Any) -> List[Any]: ...
    @classmethod
-    def from_queryset(
-        cls: Type[_Self], queryset_class: Type[QuerySet], class_name: Optional[str] = ...
-    ) -> Type[_Self]: ...
+    def from_queryset(cls, queryset_class: Type[QuerySet], class_name: Optional[str] = ...) -> Any: ...
    @classmethod
    def _get_queryset_methods(cls, queryset_class: type) -> Dict[str, Any]: ...
    def contribute_to_class(self, model: Type[Model], name: str) -> None: ...
--- a/django-stubs/template/context.pyi
+++ b/django-stubs/template/context.pyi
@@ -1,4 +1,4 @@
-from typing import Any, Callable, Dict, Iterator, List, Optional, Type, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, Type, Union, Iterable

 from django.http.request import HttpRequest
 from django.template.base import Node, Origin, Template
@@ -15,10 +15,10 @@ class ContextDict(dict):
    def __enter__(self) -> ContextDict: ...
    def __exit__(self, *args: Any, **kwargs: Any) -> None: ...

-class BaseContext:
+class BaseContext(Iterable[Any]):
    def __init__(self, dict_: Any = ...) -> None: ...
    def __copy__(self) -> BaseContext: ...
-    def __iter__(self) -> None: ...
+    def __iter__(self) -> Iterator[Any]: ...
    def push(self, *args: Any, **kwargs: Any) -> ContextDict: ...
    def pop(self) -> ContextDict: ...
    def __setitem__(self, key: Union[Node, str], value: Any) -> None: ...
@@ -50,7 +50,6 @@ class Context(BaseContext):
 class RenderContext(BaseContext):
    dicts: List[Dict[Union[IncludeNode, str], str]]
    template: Optional[Template] = ...
-    def __iter__(self) -> None: ...
    def push_state(self, template: Template, isolated_context: bool = ...) -> Iterator[None]: ...

 class RequestContext(Context):
--- a/mypy_django_plugin/helpers.py
+++ b/mypy_django_plugin/helpers.py
@@ -2,13 +2,14 @@ import typing
 from typing import Dict, Optional

 from mypy.checker import TypeChecker
-from mypy.nodes import AssignmentStmt, ClassDef, Expression, FuncDef, ImportedName, Lvalue, MypyFile, NameExpr, SymbolNode, \
+from mypy.nodes import AssignmentStmt, ClassDef, Expression, ImportedName, Lvalue, MypyFile, NameExpr, SymbolNode, \
    TypeInfo
 from mypy.plugin import FunctionContext
-from mypy.types import AnyType, CallableType, Instance, NoneTyp, Type, TypeOfAny, TypeVarType, UnionType
+from mypy.types import AnyType, Instance, NoneTyp, Type, TypeOfAny, TypeVarType, UnionType

 MODEL_CLASS_FULLNAME = 'django.db.models.base.Model'
 FIELD_FULLNAME = 'django.db.models.fields.Field'
+CHAR_FIELD_FULLNAME = 'django.db.models.fields.CharField'
 ARRAY_FIELD_FULLNAME = 'django.contrib.postgres.fields.array.ArrayField'
 AUTO_FIELD_FULLNAME = 'django.db.models.fields.AutoField'
 GENERIC_FOREIGN_KEY_FULLNAME = 'django.contrib.contenttypes.fields.GenericForeignKey'
@@ -263,9 +264,12 @@ def is_optional(typ: Type) -> bool:
    return any([isinstance(item, NoneTyp) for item in typ.items])


-
 def has_any_of_bases(info: TypeInfo, bases: typing.Sequence[str]) -> bool:
    for base_fullname in bases:
        if info.has_base(base_fullname):
            return True
    return False
+
+
+def is_none_expr(expr: Expression) -> bool:
+    return isinstance(expr, NameExpr) and expr.fullname == 'builtins.None'
--- a/mypy_django_plugin/transformers/fields.py
+++ b/mypy_django_plugin/transformers/fields.py
@@ -106,6 +106,9 @@ def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is
 def set_descriptor_types_for_field(ctx: FunctionContext) -> Instance:
    default_return_type = cast(Instance, ctx.default_return_type)
    is_nullable = helpers.parse_bool(helpers.get_argument_by_name(ctx, 'null'))
+    if not is_nullable and default_return_type.type.has_base(helpers.CHAR_FIELD_FULLNAME):
+        # blank=True for CharField can be interpreted as null=True
+        is_nullable = helpers.parse_bool(helpers.get_argument_by_name(ctx, 'blank'))

    set_type = get_private_descriptor_type(default_return_type.type, '_pyi_private_set_type',
                                           is_nullable=is_nullable)
@@ -197,3 +200,8 @@ def record_field_properties_into_outer_model_class(ctx: FunctionContext) -> None
    if blank_arg:
        is_blankable = helpers.parse_bool(blank_arg)
    fields_metadata[field_name]['blank'] = is_blankable
+
+    # default
+    default_arg = helpers.get_argument_by_name(ctx, 'default')
+    if default_arg and not helpers.is_none_expr(default_arg):
+        fields_metadata[field_name]['default_specified'] = True
--- a/mypy_django_plugin/transformers/init_create.py
+++ b/mypy_django_plugin/transformers/init_create.py
@@ -25,12 +25,13 @@ def redefine_and_typecheck_model_init(ctx: FunctionContext) -> Type:
    api = cast(TypeChecker, ctx.api)
    model: TypeInfo = ctx.default_return_type.type

-    expected_types = extract_expected_types(ctx, model)
-    # order is preserved, can use for positionals
+    expected_types = extract_expected_types(ctx, model, is_init=True)
+
+    # order is preserved, can be used for positionals
    positional_names = list(expected_types.keys())
    positional_names.remove('pk')
-    visited_positionals = set()

+    visited_positionals = set()
    # check positionals
    for i, (_, actual_pos_type) in enumerate(zip(ctx.arg_names[0], ctx.arg_types[0])):
        actual_pos_name = positional_names[i]
@@ -111,7 +112,8 @@ def extract_choices_type(model: TypeInfo, field_name: str) -> Optional[str]:
    return None


-def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, Type]:
+def extract_expected_types(ctx: FunctionContext, model: TypeInfo,
+                           is_init: bool = False) -> Dict[str, Type]:
    api = cast(TypeChecker, ctx.api)

    expected_types: Dict[str, Type] = {}
@@ -119,7 +121,11 @@ def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, T
    if not primary_key_type:
        # no explicit primary key, set pk to Any and add id
        primary_key_type = AnyType(TypeOfAny.special_form)
+        if is_init:
+            expected_types['id'] = helpers.make_optional(ctx.api.named_generic_type('builtins.int', []))
+        else:
            expected_types['id'] = ctx.api.named_generic_type('builtins.int', [])
+
    expected_types['pk'] = primary_key_type

    for base in model.mro:
@@ -141,8 +147,9 @@ def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, T
                    if field_type is None:
                        continue

-                    if typ.type.fullname() in {helpers.FOREIGN_KEY_FULLNAME, helpers.ONETOONE_FIELD_FULLNAME}:
-                        primary_key_type = AnyType(TypeOfAny.implementation_artifact)
+                    if helpers.has_any_of_bases(typ.type, (helpers.FOREIGN_KEY_FULLNAME,
+                                                           helpers.ONETOONE_FIELD_FULLNAME)):
+                        related_primary_key_type = AnyType(TypeOfAny.implementation_artifact)
                        # in case it's optional, we need Instance type
                        referred_to_model = typ.args[1]
                        is_nullable = helpers.is_optional(referred_to_model)
@@ -156,11 +163,24 @@ def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, T
                                autofield_info = api.lookup_typeinfo('django.db.models.fields.AutoField')
                                pk_type = get_private_descriptor_type(autofield_info, '_pyi_private_set_type',
                                                                      is_nullable=is_nullable)
-                            primary_key_type = pk_type
+                            related_primary_key_type = pk_type

-                        expected_types[name + '_id'] = primary_key_type
+                        if is_init:
+                            related_primary_key_type = helpers.make_optional(related_primary_key_type)

+                        expected_types[name + '_id'] = related_primary_key_type
+
+                    field_metadata = get_fields_metadata(model).get(name, {})
                    if field_type:
+                        # related fields could be None in __init__ (but should be specified before save())
+                        if helpers.has_any_of_bases(typ.type, (helpers.FOREIGN_KEY_FULLNAME,
+                                                               helpers.ONETOONE_FIELD_FULLNAME)) and is_init:
+                            field_type = helpers.make_optional(field_type)
+
+                        # if primary_key=True and default specified
+                        elif field_metadata.get('primary_key', False) and field_metadata.get('default_specified', False):
+                            field_type = helpers.make_optional(field_type)
+
                        expected_types[name] = field_type

    return expected_types
--- a/scripts/mypy.ini
+++ b/scripts/mypy.ini
@@ -1,11 +1,12 @@
 [mypy]
-strict_optional = False
+strict_optional = True
 ignore_missing_imports = True
 check_untyped_defs = True
 warn_no_return = False
 show_traceback = True
 warn_redundant_casts = True
 allow_redefinition = True
+incremental = False

 plugins =
    mypy_django_plugin.main
--- a/scripts/typecheck_tests.py
+++ b/scripts/typecheck_tests.py
@@ -88,7 +88,9 @@ IGNORED_ERRORS = {
        'Incompatible types in assignment (expression has type "QuerySet[Any]", variable has type "List[Any]")',
        '"as_sql" undefined in superclass',
        'Incompatible types in assignment (expression has type "FlatValuesListIterable", '
-        + 'variable has type "ValuesListIterable")'
+        + 'variable has type "ValuesListIterable")',
+        'Incompatible type for "contact" of "Book" (got "Optional[Author]", expected "Union[Author, Combinable]")',
+        'Incompatible type for "publisher" of "Book" (got "Optional[Publisher]", expected "Union[Publisher, Combinable]")'
    ],
    'aggregation_regress': [
        'Incompatible types in assignment (expression has type "List[str]", variable has type "QuerySet[Author]")',
@@ -188,6 +190,9 @@ IGNORED_ERRORS = {
    'logging_tests': [
        re.compile('"(setUpClass|tearDownClass)" undefined in superclass')
    ],
+    'many_to_one': [
+        'Incompatible type for "parent" of "Child" (got "None", expected "Union[Parent, Combinable]")'
+    ],
    'model_inheritance_regress': [
        'Incompatible types in assignment (expression has type "List[Supplier]", variable has type "QuerySet[Supplier]")'
    ],
@@ -200,7 +205,8 @@ IGNORED_ERRORS = {
        'Unexpected keyword argument "name" for "Person"',
        'Cannot assign multiple types to name "PersonTwoImages" without an explicit "Type[...]" annotation',
        'Incompatible types in assignment (expression has type "Type[Person]", '
-        + 'base class "ImageFieldTestMixin" defined the type as "Type[PersonWithHeightAndWidth]")'
+        + 'base class "ImageFieldTestMixin" defined the type as "Type[PersonWithHeightAndWidth]")',
+        'note: "Person" defined here'
    ],
    'model_regress': [
        'Too many arguments for "Worker"',
@@ -251,7 +257,8 @@ IGNORED_ERRORS = {
        re.compile('Unexpected attribute "(full_name|full_name_2)" for model "Person"')
    ],
    'queries': [
-        'Incompatible types in assignment (expression has type "None", variable has type "str")'
+        'Incompatible types in assignment (expression has type "None", variable has type "str")',
+        'Invalid index type "Optional[str]" for "Dict[str, int]"; expected type "str"'
    ],
    'requests': [
        'Incompatible types in assignment (expression has type "Dict[str, str]", variable has type "QueryDict")'
@@ -265,7 +272,8 @@ IGNORED_ERRORS = {
        'has no attribute "read_by"'
    ],
    'signals': [
-        'Argument 1 to "append" of "list" has incompatible type "Tuple[Any, Any, Any, Any]"; expected "Tuple[Any, Any, Any]"'
+        'Argument 1 to "append" of "list" has incompatible type "Tuple[Any, Any, Optional[Any], Any]"; '
+        + 'expected "Tuple[Any, Any, Any]"'
    ],
    'syndication_tests': [
        'List or tuple expected as variable arguments'
@@ -277,7 +285,8 @@ IGNORED_ERRORS = {
        'Value of type "object" is not indexable'
    ],
    'schema': [
-        'Incompatible type for "info" of "Note" (got "None", expected "Union[str, Combinable]")'
+        'Incompatible type for "info" of "Note" (got "None", expected "Union[str, Combinable]")',
+        'Incompatible type for "detail_info" of "NoteRename" (got "None", expected "Union[str, Combinable]")'
    ],
    'settings_tests': [
        'Argument 1 to "Settings" has incompatible type "Optional[str]"; expected "str"'
@@ -290,7 +299,9 @@ IGNORED_ERRORS = {
        'Incompatible types in assignment (expression has type "HttpResponse", variable has type "StreamingHttpResponse")'
    ],
    'test_client_regress': [
-        'Incompatible types in assignment (expression has type "Dict[<nothing>, <nothing>]", variable has type "SessionBase")'
+        'Incompatible types in assignment (expression has type "Dict[<nothing>, <nothing>]", variable has type "SessionBase")',
+        'Unsupported left operand type for + ("None")',
+        'Both left and right operands are unions'
    ],
    'timezones': [
        'Too few arguments for "render" of "Template"'
--- a/test-data/plugins.ini
+++ b/test-data/plugins.ini
@@ -1,5 +1,5 @@
 [mypy]
-incremental = True
+incremental = False
 strict_optional = True
 plugins =
    mypy_django_plugin.main
--- a/test-data/typecheck/fields.test
+++ b/test-data/typecheck/fields.test
@@ -104,3 +104,10 @@ class MyModel(ParentModel):
    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
 reveal_type(MyModel().id)  # E: Revealed type is 'uuid.UUID*'
 [out]
+
+[CASE blank_for_charfield_is_the_same_as_null]
+from django.db import models
+class MyModel(models.Model):
+    text = models.CharField(max_length=30, blank=True)
+MyModel(text=None)
+[out]
--- a/test-data/typecheck/model_create.test
+++ b/test-data/typecheck/model_create.test
@@ -33,3 +33,32 @@ class Child4(Child1):
    value4 = models.IntegerField()
 Child4.objects.create(name1='n1', name2='n2', value=1, value4=4)
 [out]
+
+[CASE optional_primary_key_for_create_is_error]
+from django.db import models
+class MyModel(models.Model):
+    pass
+MyModel.objects.create(id=None)  # E: Incompatible type for "id" of "MyModel" (got "None", expected "int")
+
+[CASE optional_related_model_for_create_is_error]
+from django.db import models
+class Publisher(models.Model):
+    pass
+class Book(models.Model):
+    publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE)
+Book.objects.create(publisher=None)  # E: Incompatible type for "publisher" of "Book" (got "None", expected "Union[Publisher, Combinable]")
+
+[CASE when_default_for_primary_key_is_specified_allow_none_to_be_set]
+from django.db import models
+def return_int():
+    return 0
+class MyModel(models.Model):
+    id = models.IntegerField(primary_key=True, default=return_int)
+MyModel(id=None)
+MyModel.objects.create(id=None)
+
+class MyModel2(models.Model):
+    id = models.IntegerField(primary_key=True, default=None)
+MyModel2(id=None)  # E: Incompatible type for "id" of "MyModel2" (got "None", expected "Union[int, Combinable, Literal['']]")
+MyModel2.objects.create(id=None)  # E: Incompatible type for "id" of "MyModel2" (got "None", expected "Union[int, Combinable, Literal['']]")
+[out]
--- a/test-data/typecheck/model_init.test
+++ b/test-data/typecheck/model_init.test
@@ -78,8 +78,8 @@ class Book(models.Model):
    publisher_dt = models.ForeignKey(PublisherDatetime, on_delete=models.CASCADE)

 Book(publisher_id=1)
-Book(publisher_id=[])  # E: Incompatible type for "publisher_id" of "Book" (got "List[Any]", expected "Union[Combinable, int, str]")
-Book(publisher_dt_id=11)  # E: Incompatible type for "publisher_dt_id" of "Book" (got "int", expected "Union[str, date, datetime, Combinable]")
+Book(publisher_id=[])  # E: Incompatible type for "publisher_id" of "Book" (got "List[Any]", expected "Union[Combinable, int, str, None]")
+Book(publisher_dt_id=11)  # E: Incompatible type for "publisher_dt_id" of "Book" (got "int", expected "Union[str, date, Combinable, None]")
 [out]

 [CASE setting_value_to_an_array_of_ints]
@@ -158,3 +158,20 @@ InvoiceRow.objects.create(base_amount=Decimal(0), vat_rate=Decimal(0))
 main:3: error: Cannot find module named 'fields2'
 main:3: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports

+[CASE optional_primary_key_is_allowed_for_init]
+from django.db import models
+class MyModel(models.Model):
+    pass
+MyModel(id=None)
+MyModel(None)
+[out]
+
+[CASE optional_related_model_is_allowed_for_init]
+from django.db import models
+class Publisher(models.Model):
+    pass
+class Book(models.Model):
+    publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE)
+Book(publisher=None)
+Book(publisher_id=None)
+[out]
--- a/test-data/typecheck/related_fields.test
+++ b/test-data/typecheck/related_fields.test
@@ -260,7 +260,7 @@ class Book(models.Model):
 reveal_type(Book().publisher_id)  # E: Revealed type is 'builtins.str'
 Book(publisher_id=1)
 Book(publisher_id='hello')
-Book(publisher_id=datetime.datetime.now())  # E: Incompatible type for "publisher_id" of "Book" (got "datetime", expected "Union[str, int, Combinable]")
+Book(publisher_id=datetime.datetime.now())  # E: Incompatible type for "publisher_id" of "Book" (got "datetime", expected "Union[str, int, Combinable, None]")
 Book.objects.create(publisher_id=1)
 Book.objects.create(publisher_id='hello')

@@ -271,7 +271,7 @@ class Book2(models.Model):

 reveal_type(Book2().publisher_id)  # E: Revealed type is 'builtins.int'
 Book2(publisher_id=1)
-Book2(publisher_id='hello')  # E: Incompatible type for "publisher_id" of "Book2" (got "str", expected "Union[int, Combinable, Literal['']]")
+Book2(publisher_id='hello')  # E: Incompatible type for "publisher_id" of "Book2" (got "str", expected "Union[int, Combinable, Literal[''], None]")
 Book2.objects.create(publisher_id=1)
 Book2.objects.create(publisher_id='hello')  # E: Incompatible type for "publisher_id" of "Book2" (got "str", expected "Union[int, Combinable, Literal['']]")
 [out]