finish strict_optional support, enable it for typechecking of django tests

django-stubs/apps/config.pyi

@@ -12,7 +12,7 @@ class AppConfig:
     verbose_name: str = ...
     path: str = ...
     models_module: None = ...
-    models: Optional[Dict[str, Type[Model]]] = ...
+    models: Dict[str, Type[Model]] = ...
     def __init__(self, app_name: str, app_module: Optional[Any]) -> None: ...
     @classmethod
     def create(cls, entry: str) -> AppConfig: ...

django-stubs/contrib/admin/options.pyi

@@ -57,9 +57,15 @@ class BaseModelAdmin:
     checks_class: Any = ...
     def check(self, **kwargs: Any) -> List[Union[str, Error]]: ...
     def __init__(self) -> None: ...
-    def formfield_for_dbfield(self, db_field: Field, request: WSGIRequest, **kwargs: Any) -> Optional[Field]: ...
+    def formfield_for_dbfield(
-    def formfield_for_choice_field(self, db_field: Field, request: WSGIRequest, **kwargs: Any) -> TypedChoiceField: ...
+        self, db_field: Field, request: Optional[WSGIRequest], **kwargs: Any
-    def get_field_queryset(self, db: None, db_field: RelatedField, request: WSGIRequest) -> Optional[QuerySet]: ...
+    ) -> Optional[Field]: ...
+    def formfield_for_choice_field(
+        self, db_field: Field, request: Optional[WSGIRequest], **kwargs: Any
+    ) -> TypedChoiceField: ...
+    def get_field_queryset(
+        self, db: None, db_field: RelatedField, request: Optional[WSGIRequest]
+    ) -> Optional[QuerySet]: ...
     def formfield_for_foreignkey(
         self, db_field: ForeignKey, request: Optional[WSGIRequest], **kwargs: Any
     ) -> Optional[ModelChoiceField]: ...
@@ -90,7 +96,7 @@ class BaseModelAdmin:
 class ModelAdmin(BaseModelAdmin):
     formfield_overrides: Any
     list_display: Sequence[Union[str, Callable]] = ...
-    list_display_links: Sequence[Union[str, Callable]] = ...
+    list_display_links: Optional[Sequence[Union[str, Callable]]] = ...
     list_filter: Sequence[Union[str, Type[ListFilter], Tuple[str, Type[ListFilter]]]] = ...
     list_select_related: Union[bool, Sequence[str]] = ...
     list_per_page: int = ...

django-stubs/db/migrations/state.pyi

@@ -22,9 +22,9 @@ class ModelState:
     name: str
     app_label: str
     fields: List[Tuple[str, Field]]
-    options: Optional[Dict[str, Any]] = ...
+    options: Dict[str, Any] = ...
-    bases: Optional[Tuple[Type[Model]]] = ...
+    bases: Tuple[Type[Model]] = ...
-    managers: Optional[List[Tuple[str, Manager]]] = ...
+    managers: List[Tuple[str, Manager]] = ...
     def __init__(
         self,
         app_label: str,

django-stubs/db/models/fields/__init__.pyi

@@ -224,7 +224,7 @@ class GenericIPAddressField(Field[_ST, _GT]):
 class DateTimeCheckMixin: ...
 
 class DateField(DateTimeCheckMixin, Field[_ST, _GT]):
-    _pyi_private_set_type: Union[str, date, datetime, Combinable]
+    _pyi_private_set_type: Union[str, date, Combinable]
     _pyi_private_get_type: date
     def __init__(
         self,

django-stubs/db/models/manager.pyi

@@ -4,7 +4,6 @@ from django.db.models.base import Model
 from django.db.models.query import QuerySet
 
 _T = TypeVar("_T", bound=Model, covariant=True)
-_Self = TypeVar("_Self", bound="BaseManager")
 
 class BaseManager(QuerySet[_T]):
     creation_counter: int = ...
@@ -17,9 +16,7 @@ class BaseManager(QuerySet[_T]):
     def deconstruct(self) -> Tuple[bool, str, None, Tuple, Dict[str, int]]: ...
     def check(self, **kwargs: Any) -> List[Any]: ...
     @classmethod
-    def from_queryset(
+    def from_queryset(cls, queryset_class: Type[QuerySet], class_name: Optional[str] = ...) -> Any: ...
-        cls: Type[_Self], queryset_class: Type[QuerySet], class_name: Optional[str] = ...
-    ) -> Type[_Self]: ...
     @classmethod
     def _get_queryset_methods(cls, queryset_class: type) -> Dict[str, Any]: ...
     def contribute_to_class(self, model: Type[Model], name: str) -> None: ...

django-stubs/template/context.pyi

@@ -1,4 +1,4 @@
-from typing import Any, Callable, Dict, Iterator, List, Optional, Type, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, Type, Union, Iterable
 
 from django.http.request import HttpRequest
 from django.template.base import Node, Origin, Template
@@ -15,10 +15,10 @@ class ContextDict(dict):
     def __enter__(self) -> ContextDict: ...
     def __exit__(self, *args: Any, **kwargs: Any) -> None: ...
 
-class BaseContext:
+class BaseContext(Iterable[Any]):
     def __init__(self, dict_: Any = ...) -> None: ...
     def __copy__(self) -> BaseContext: ...
-    def __iter__(self) -> None: ...
+    def __iter__(self) -> Iterator[Any]: ...
     def push(self, *args: Any, **kwargs: Any) -> ContextDict: ...
     def pop(self) -> ContextDict: ...
     def __setitem__(self, key: Union[Node, str], value: Any) -> None: ...
@@ -50,7 +50,6 @@ class Context(BaseContext):
 class RenderContext(BaseContext):
     dicts: List[Dict[Union[IncludeNode, str], str]]
     template: Optional[Template] = ...
-    def __iter__(self) -> None: ...
     def push_state(self, template: Template, isolated_context: bool = ...) -> Iterator[None]: ...
 
 class RequestContext(Context):

mypy_django_plugin/helpers.py

@@ -2,13 +2,14 @@ import typing
 from typing import Dict, Optional
 
 from mypy.checker import TypeChecker
-from mypy.nodes import AssignmentStmt, ClassDef, Expression, FuncDef, ImportedName, Lvalue, MypyFile, NameExpr, SymbolNode, \
+from mypy.nodes import AssignmentStmt, ClassDef, Expression, ImportedName, Lvalue, MypyFile, NameExpr, SymbolNode, \
     TypeInfo
 from mypy.plugin import FunctionContext
-from mypy.types import AnyType, CallableType, Instance, NoneTyp, Type, TypeOfAny, TypeVarType, UnionType
+from mypy.types import AnyType, Instance, NoneTyp, Type, TypeOfAny, TypeVarType, UnionType
 
 MODEL_CLASS_FULLNAME = 'django.db.models.base.Model'
 FIELD_FULLNAME = 'django.db.models.fields.Field'
+CHAR_FIELD_FULLNAME = 'django.db.models.fields.CharField'
 ARRAY_FIELD_FULLNAME = 'django.contrib.postgres.fields.array.ArrayField'
 AUTO_FIELD_FULLNAME = 'django.db.models.fields.AutoField'
 GENERIC_FOREIGN_KEY_FULLNAME = 'django.contrib.contenttypes.fields.GenericForeignKey'
@@ -263,9 +264,12 @@ def is_optional(typ: Type) -> bool:
     return any([isinstance(item, NoneTyp) for item in typ.items])
 
 
-
 def has_any_of_bases(info: TypeInfo, bases: typing.Sequence[str]) -> bool:
     for base_fullname in bases:
         if info.has_base(base_fullname):
             return True
     return False
+
+
+def is_none_expr(expr: Expression) -> bool:
+    return isinstance(expr, NameExpr) and expr.fullname == 'builtins.None'

mypy_django_plugin/transformers/fields.py

@@ -106,6 +106,9 @@ def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is
 def set_descriptor_types_for_field(ctx: FunctionContext) -> Instance:
     default_return_type = cast(Instance, ctx.default_return_type)
     is_nullable = helpers.parse_bool(helpers.get_argument_by_name(ctx, 'null'))
+    if not is_nullable and default_return_type.type.has_base(helpers.CHAR_FIELD_FULLNAME):
+        # blank=True for CharField can be interpreted as null=True
+        is_nullable = helpers.parse_bool(helpers.get_argument_by_name(ctx, 'blank'))
 
     set_type = get_private_descriptor_type(default_return_type.type, '_pyi_private_set_type',
                                            is_nullable=is_nullable)
@@ -197,3 +200,8 @@ def record_field_properties_into_outer_model_class(ctx: FunctionContext) -> None
     if blank_arg:
         is_blankable = helpers.parse_bool(blank_arg)
     fields_metadata[field_name]['blank'] = is_blankable
+
+    # default
+    default_arg = helpers.get_argument_by_name(ctx, 'default')
+    if default_arg and not helpers.is_none_expr(default_arg):
+        fields_metadata[field_name]['default_specified'] = True

mypy_django_plugin/transformers/init_create.py

@@ -25,12 +25,13 @@ def redefine_and_typecheck_model_init(ctx: FunctionContext) -> Type:
     api = cast(TypeChecker, ctx.api)
     model: TypeInfo = ctx.default_return_type.type
 
-    expected_types = extract_expected_types(ctx, model)
+    expected_types = extract_expected_types(ctx, model, is_init=True)
-    # order is preserved, can use for positionals
+
+    # order is preserved, can be used for positionals
     positional_names = list(expected_types.keys())
     positional_names.remove('pk')
-    visited_positionals = set()
 
+    visited_positionals = set()
     # check positionals
     for i, (_, actual_pos_type) in enumerate(zip(ctx.arg_names[0], ctx.arg_types[0])):
         actual_pos_name = positional_names[i]
@@ -111,7 +112,8 @@ def extract_choices_type(model: TypeInfo, field_name: str) -> Optional[str]:
     return None
 
 
-def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, Type]:
+def extract_expected_types(ctx: FunctionContext, model: TypeInfo,
+                           is_init: bool = False) -> Dict[str, Type]:
     api = cast(TypeChecker, ctx.api)
 
     expected_types: Dict[str, Type] = {}
@@ -119,7 +121,11 @@ def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, T
     if not primary_key_type:
         # no explicit primary key, set pk to Any and add id
         primary_key_type = AnyType(TypeOfAny.special_form)
+        if is_init:
+            expected_types['id'] = helpers.make_optional(ctx.api.named_generic_type('builtins.int', []))
+        else:
             expected_types['id'] = ctx.api.named_generic_type('builtins.int', [])
+
     expected_types['pk'] = primary_key_type
 
     for base in model.mro:
@@ -141,8 +147,9 @@ def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, T
                     if field_type is None:
                         continue
 
-                    if typ.type.fullname() in {helpers.FOREIGN_KEY_FULLNAME, helpers.ONETOONE_FIELD_FULLNAME}:
+                    if helpers.has_any_of_bases(typ.type, (helpers.FOREIGN_KEY_FULLNAME,
-                        primary_key_type = AnyType(TypeOfAny.implementation_artifact)
+                                                           helpers.ONETOONE_FIELD_FULLNAME)):
+                        related_primary_key_type = AnyType(TypeOfAny.implementation_artifact)
                         # in case it's optional, we need Instance type
                         referred_to_model = typ.args[1]
                         is_nullable = helpers.is_optional(referred_to_model)
@@ -156,11 +163,24 @@ def extract_expected_types(ctx: FunctionContext, model: TypeInfo) -> Dict[str, T
                                 autofield_info = api.lookup_typeinfo('django.db.models.fields.AutoField')
                                 pk_type = get_private_descriptor_type(autofield_info, '_pyi_private_set_type',
                                                                       is_nullable=is_nullable)
-                            primary_key_type = pk_type
+                            related_primary_key_type = pk_type
 
-                        expected_types[name + '_id'] = primary_key_type
+                        if is_init:
+                            related_primary_key_type = helpers.make_optional(related_primary_key_type)
 
+                        expected_types[name + '_id'] = related_primary_key_type
+
+                    field_metadata = get_fields_metadata(model).get(name, {})
                     if field_type:
+                        # related fields could be None in __init__ (but should be specified before save())
+                        if helpers.has_any_of_bases(typ.type, (helpers.FOREIGN_KEY_FULLNAME,
+                                                               helpers.ONETOONE_FIELD_FULLNAME)) and is_init:
+                            field_type = helpers.make_optional(field_type)
+
+                        # if primary_key=True and default specified
+                        elif field_metadata.get('primary_key', False) and field_metadata.get('default_specified', False):
+                            field_type = helpers.make_optional(field_type)
+
                         expected_types[name] = field_type
 
     return expected_types

scripts/mypy.ini

@@ -1,11 +1,12 @@
 [mypy]
-strict_optional = False
+strict_optional = True
 ignore_missing_imports = True
 check_untyped_defs = True
 warn_no_return = False
 show_traceback = True
 warn_redundant_casts = True
 allow_redefinition = True
+incremental = False
 
 plugins =
     mypy_django_plugin.main

scripts/typecheck_tests.py

@@ -88,7 +88,9 @@ IGNORED_ERRORS = {
         'Incompatible types in assignment (expression has type "QuerySet[Any]", variable has type "List[Any]")',
         '"as_sql" undefined in superclass',
         'Incompatible types in assignment (expression has type "FlatValuesListIterable", '
-        + 'variable has type "ValuesListIterable")'
+        + 'variable has type "ValuesListIterable")',
+        'Incompatible type for "contact" of "Book" (got "Optional[Author]", expected "Union[Author, Combinable]")',
+        'Incompatible type for "publisher" of "Book" (got "Optional[Publisher]", expected "Union[Publisher, Combinable]")'
     ],
     'aggregation_regress': [
         'Incompatible types in assignment (expression has type "List[str]", variable has type "QuerySet[Author]")',
@@ -188,6 +190,9 @@ IGNORED_ERRORS = {
     'logging_tests': [
         re.compile('"(setUpClass|tearDownClass)" undefined in superclass')
     ],
+    'many_to_one': [
+        'Incompatible type for "parent" of "Child" (got "None", expected "Union[Parent, Combinable]")'
+    ],
     'model_inheritance_regress': [
         'Incompatible types in assignment (expression has type "List[Supplier]", variable has type "QuerySet[Supplier]")'
     ],
@@ -200,7 +205,8 @@ IGNORED_ERRORS = {
         'Unexpected keyword argument "name" for "Person"',
         'Cannot assign multiple types to name "PersonTwoImages" without an explicit "Type[...]" annotation',
         'Incompatible types in assignment (expression has type "Type[Person]", '
-        + 'base class "ImageFieldTestMixin" defined the type as "Type[PersonWithHeightAndWidth]")'
+        + 'base class "ImageFieldTestMixin" defined the type as "Type[PersonWithHeightAndWidth]")',
+        'note: "Person" defined here'
     ],
     'model_regress': [
         'Too many arguments for "Worker"',
@@ -251,7 +257,8 @@ IGNORED_ERRORS = {
         re.compile('Unexpected attribute "(full_name|full_name_2)" for model "Person"')
     ],
     'queries': [
-        'Incompatible types in assignment (expression has type "None", variable has type "str")'
+        'Incompatible types in assignment (expression has type "None", variable has type "str")',
+        'Invalid index type "Optional[str]" for "Dict[str, int]"; expected type "str"'
     ],
     'requests': [
         'Incompatible types in assignment (expression has type "Dict[str, str]", variable has type "QueryDict")'
@@ -265,7 +272,8 @@ IGNORED_ERRORS = {
         'has no attribute "read_by"'
     ],
     'signals': [
-        'Argument 1 to "append" of "list" has incompatible type "Tuple[Any, Any, Any, Any]"; expected "Tuple[Any, Any, Any]"'
+        'Argument 1 to "append" of "list" has incompatible type "Tuple[Any, Any, Optional[Any], Any]"; '
+        + 'expected "Tuple[Any, Any, Any]"'
     ],
     'syndication_tests': [
         'List or tuple expected as variable arguments'
@@ -277,7 +285,8 @@ IGNORED_ERRORS = {
         'Value of type "object" is not indexable'
     ],
     'schema': [
-        'Incompatible type for "info" of "Note" (got "None", expected "Union[str, Combinable]")'
+        'Incompatible type for "info" of "Note" (got "None", expected "Union[str, Combinable]")',
+        'Incompatible type for "detail_info" of "NoteRename" (got "None", expected "Union[str, Combinable]")'
     ],
     'settings_tests': [
         'Argument 1 to "Settings" has incompatible type "Optional[str]"; expected "str"'
@@ -290,7 +299,9 @@ IGNORED_ERRORS = {
         'Incompatible types in assignment (expression has type "HttpResponse", variable has type "StreamingHttpResponse")'
     ],
     'test_client_regress': [
-        'Incompatible types in assignment (expression has type "Dict[<nothing>, <nothing>]", variable has type "SessionBase")'
+        'Incompatible types in assignment (expression has type "Dict[<nothing>, <nothing>]", variable has type "SessionBase")',
+        'Unsupported left operand type for + ("None")',
+        'Both left and right operands are unions'
     ],
     'timezones': [
         'Too few arguments for "render" of "Template"'

test-data/plugins.ini

@@ -1,5 +1,5 @@
 [mypy]
-incremental = True
+incremental = False
 strict_optional = True
 plugins =
     mypy_django_plugin.main

test-data/typecheck/fields.test

@@ -104,3 +104,10 @@ class MyModel(ParentModel):
     id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
 reveal_type(MyModel().id)  # E: Revealed type is 'uuid.UUID*'
 [out]
+
+[CASE blank_for_charfield_is_the_same_as_null]
+from django.db import models
+class MyModel(models.Model):
+    text = models.CharField(max_length=30, blank=True)
+MyModel(text=None)
+[out]

test-data/typecheck/model_create.test

@@ -33,3 +33,32 @@ class Child4(Child1):
     value4 = models.IntegerField()
 Child4.objects.create(name1='n1', name2='n2', value=1, value4=4)
 [out]
+
+[CASE optional_primary_key_for_create_is_error]
+from django.db import models
+class MyModel(models.Model):
+    pass
+MyModel.objects.create(id=None)  # E: Incompatible type for "id" of "MyModel" (got "None", expected "int")
+
+[CASE optional_related_model_for_create_is_error]
+from django.db import models
+class Publisher(models.Model):
+    pass
+class Book(models.Model):
+    publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE)
+Book.objects.create(publisher=None)  # E: Incompatible type for "publisher" of "Book" (got "None", expected "Union[Publisher, Combinable]")
+
+[CASE when_default_for_primary_key_is_specified_allow_none_to_be_set]
+from django.db import models
+def return_int():
+    return 0
+class MyModel(models.Model):
+    id = models.IntegerField(primary_key=True, default=return_int)
+MyModel(id=None)
+MyModel.objects.create(id=None)
+
+class MyModel2(models.Model):
+    id = models.IntegerField(primary_key=True, default=None)
+MyModel2(id=None)  # E: Incompatible type for "id" of "MyModel2" (got "None", expected "Union[int, Combinable, Literal['']]")
+MyModel2.objects.create(id=None)  # E: Incompatible type for "id" of "MyModel2" (got "None", expected "Union[int, Combinable, Literal['']]")
+[out]

test-data/typecheck/model_init.test

@@ -78,8 +78,8 @@ class Book(models.Model):
     publisher_dt = models.ForeignKey(PublisherDatetime, on_delete=models.CASCADE)
 
 Book(publisher_id=1)
-Book(publisher_id=[])  # E: Incompatible type for "publisher_id" of "Book" (got "List[Any]", expected "Union[Combinable, int, str]")
+Book(publisher_id=[])  # E: Incompatible type for "publisher_id" of "Book" (got "List[Any]", expected "Union[Combinable, int, str, None]")
-Book(publisher_dt_id=11)  # E: Incompatible type for "publisher_dt_id" of "Book" (got "int", expected "Union[str, date, datetime, Combinable]")
+Book(publisher_dt_id=11)  # E: Incompatible type for "publisher_dt_id" of "Book" (got "int", expected "Union[str, date, Combinable, None]")
 [out]
 
 [CASE setting_value_to_an_array_of_ints]
@@ -158,3 +158,20 @@ InvoiceRow.objects.create(base_amount=Decimal(0), vat_rate=Decimal(0))
 main:3: error: Cannot find module named 'fields2'
 main:3: note: See https://mypy.readthedocs.io/en/latest/running_mypy.html#missing-imports
 
+[CASE optional_primary_key_is_allowed_for_init]
+from django.db import models
+class MyModel(models.Model):
+    pass
+MyModel(id=None)
+MyModel(None)
+[out]
+
+[CASE optional_related_model_is_allowed_for_init]
+from django.db import models
+class Publisher(models.Model):
+    pass
+class Book(models.Model):
+    publisher = models.ForeignKey(Publisher, on_delete=models.CASCADE)
+Book(publisher=None)
+Book(publisher_id=None)
+[out]

test-data/typecheck/related_fields.test

@@ -260,7 +260,7 @@ class Book(models.Model):
 reveal_type(Book().publisher_id)  # E: Revealed type is 'builtins.str'
 Book(publisher_id=1)
 Book(publisher_id='hello')
-Book(publisher_id=datetime.datetime.now())  # E: Incompatible type for "publisher_id" of "Book" (got "datetime", expected "Union[str, int, Combinable]")
+Book(publisher_id=datetime.datetime.now())  # E: Incompatible type for "publisher_id" of "Book" (got "datetime", expected "Union[str, int, Combinable, None]")
 Book.objects.create(publisher_id=1)
 Book.objects.create(publisher_id='hello')
 
@@ -271,7 +271,7 @@ class Book2(models.Model):
 
 reveal_type(Book2().publisher_id)  # E: Revealed type is 'builtins.int'
 Book2(publisher_id=1)
-Book2(publisher_id='hello')  # E: Incompatible type for "publisher_id" of "Book2" (got "str", expected "Union[int, Combinable, Literal['']]")
+Book2(publisher_id='hello')  # E: Incompatible type for "publisher_id" of "Book2" (got "str", expected "Union[int, Combinable, Literal[''], None]")
 Book2.objects.create(publisher_id=1)
 Book2.objects.create(publisher_id='hello')  # E: Incompatible type for "publisher_id" of "Book2" (got "str", expected "Union[int, Combinable, Literal['']]")
 [out]