Add Django 3.0 testing to CI (#246)

* add Django 3.0 testing to CI

* remove importlib_metadata usage

* conditionally load choices module for tests

.gitmodules

@@ -1,4 +0,0 @@
-[submodule "django-sources"]
-	path = django-sources
-	url = https://github.com/django/django.git
-	branch = stable/2.2.x

.travis.yml

@@ -8,13 +8,23 @@ jobs:
       python: 3.7
       script: 'pytest'
 
-    - name: Typecheck Django test suite with python 3.7
+    - name: Typecheck Django 3.0 test suite with python 3.7
       python: 3.7
-      script: 'python ./scripts/typecheck_tests.py'
+      script: |
+        pip install Django==3.0.*
+        python ./scripts/typecheck_tests.py --django_version=3.0
 
-    - name: Typecheck Django test suite with python 3.6
+    - name: Typecheck Django 3.0 test suite with python 3.6
       python: 3.6
-      script: 'python ./scripts/typecheck_tests.py'
+      script: |
+        pip install Django==3.0.*
+        python ./scripts/typecheck_tests.py --django_version=3.0
+
+    - name: Typecheck Django 2.2 test suite with python 3.7
+      python: 3.7
+      script: |
+        pip install Django==2.2.*
+        python ./scripts/typecheck_tests.py --django_version=2.2
 
     - name: Mypy for plugin code
       python: 3.7

dev-requirements.txt

@@ -4,4 +4,5 @@ psycopg2
 flake8==3.7.8
 flake8-pyi==19.3.0
 isort==4.3.21
+gitpython==3.0.5
 -e .

django-stubs/db/models/__init__.pyi

@@ -129,3 +129,5 @@ from .constraints import (
     CheckConstraint as CheckConstraint,
     UniqueConstraint as UniqueConstraint,
 )
+
+from .enums import Choices as Choices, IntegerChoices as IntegerChoices, TextChoices as TextChoices

django-stubs/db/models/enums.pyi

@@ -0,0 +1,30 @@
+import enum
+from typing import Any, List, Tuple
+
+class ChoicesMeta(enum.EnumMeta):
+    names: List[str] = ...
+    choices: List[Tuple[Any, str]] = ...
+    labels: List[str] = ...
+    values: List[Any] = ...
+    def __contains__(self, item: Any) -> bool: ...
+
+class Choices(enum.Enum, metaclass=ChoicesMeta):
+    def __str__(self): ...
+
+# fake
+class _IntegerChoicesMeta(ChoicesMeta):
+    names: List[str] = ...
+    choices: List[Tuple[int, str]] = ...
+    labels: List[str] = ...
+    values: List[int] = ...
+
+class IntegerChoices(int, Choices, metaclass=_IntegerChoicesMeta): ...
+
+# fake
+class _TextChoicesMeta(ChoicesMeta):
+    names: List[str] = ...
+    choices: List[Tuple[str, str]] = ...
+    labels: List[str] = ...
+    values: List[str] = ...
+
+class TextChoices(str, Choices, metaclass=_TextChoicesMeta): ...

django-stubs/db/models/fields/__init__.pyi

@@ -108,7 +108,8 @@ class Field(RegisterLookupMixin, Generic[_ST, _GT]):
     def db_parameters(self, connection: Any) -> Dict[str, str]: ...
     def get_prep_value(self, value: Any) -> Any: ...
     def get_internal_type(self) -> str: ...
-    def formfield(self, **kwargs) -> FormField: ...
+    # TODO: plugin support
+    def formfield(self, **kwargs) -> Any: ...
     def save_form_data(self, instance: Model, data: Any) -> None: ...
     def contribute_to_class(self, cls: Type[Model], name: str, private_only: bool = ...) -> None: ...
     def to_python(self, value: Any) -> Any: ...
@@ -361,20 +362,20 @@ class UUIDField(Field[_ST, _GT]):
     _pyi_private_get_type: uuid.UUID
 
 class FilePathField(Field[_ST, _GT]):
-    path: str = ...
-    match: Optional[Any] = ...
+    path: Any = ...
+    match: Optional[str] = ...
     recursive: bool = ...
     allow_files: bool = ...
     allow_folders: bool = ...
     def __init__(
         self,
-        verbose_name: Optional[Union[str, bytes]] = ...,
-        name: Optional[str] = ...,
-        path: str = ...,
-        match: Optional[Any] = ...,
+        path: Union[str, Callable[..., str]] = ...,
+        match: Optional[str] = ...,
         recursive: bool = ...,
         allow_files: bool = ...,
         allow_folders: bool = ...,
+        verbose_name: Optional[str] = ...,
+        name: Optional[str] = ...,
         primary_key: bool = ...,
         max_length: int = ...,
         unique: bool = ...,

django-stubs/db/models/fields/files.pyi

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Any, Callable, Iterable, List, Optional, Tuple, Type, TypeVar, Union, overload
 
 from django.core.files.base import File
@@ -39,11 +40,10 @@ class FileField(Field):
     upload_to: Union[str, Callable] = ...
     def __init__(
         self,
+        upload_to: Union[str, Callable, Path] = ...,
+        storage: Optional[Storage] = ...,
         verbose_name: Optional[Union[str, bytes]] = ...,
         name: Optional[str] = ...,
-        upload_to: Union[str, Callable] = ...,
-        storage: Optional[Storage] = ...,
-        primary_key: bool = ...,
         max_length: Optional[int] = ...,
         unique: bool = ...,
         blank: bool = ...,

django-stubs/db/models/options.pyi

@@ -108,6 +108,8 @@ class Options(Generic[_M]):
     def managers(self) -> List[Manager]: ...
     @property
     def managers_map(self) -> Dict[str, Manager]: ...
+    @property
+    def db_returning_fields(self) -> List[Field]: ...
     def get_field(self, field_name: Union[Callable, str]) -> Field: ...
     def get_base_chain(self, model: Type[Model]) -> List[Type[Model]]: ...
     def get_parent_list(self) -> List[Type[Model]]: ...

django-stubs/db/models/query_utils.pyi

@@ -42,6 +42,7 @@ class Q(tree.Node):
 
 class DeferredAttribute:
     field_name: str = ...
+    field: Field
     def __init__(self, field_name: str) -> None: ...
 
 class RegisterLookupMixin:

django-stubs/db/models/sql/query.pyi

@@ -1,6 +1,6 @@
 import collections
 from collections import OrderedDict, namedtuple
-from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Type, Union
+from typing import Any, Callable, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Type, Union, Iterable
 
 from django.db.models.lookups import Lookup, Transform
 from django.db.models.query_utils import PathInfo, RegisterLookupMixin
@@ -155,19 +155,19 @@ class Query:
     def add_ordering(self, *ordering: Any) -> None: ...
     def clear_ordering(self, force_empty: bool) -> None: ...
     def set_group_by(self) -> None: ...
-    def add_select_related(self, fields: Tuple[str]) -> None: ...
+    def add_select_related(self, fields: Iterable[str]) -> None: ...
     def add_extra(
         self,
-        select: Optional[Union[Dict[str, int], Dict[str, str], OrderedDict]],
-        select_params: Optional[Union[List[int], List[str], Tuple[int]]],
-        where: Optional[List[str]],
-        params: Optional[List[str]],
-        tables: Optional[List[str]],
-        order_by: Optional[Union[List[str], Tuple[str]]],
+        select: Optional[Dict[str, Any]],
+        select_params: Optional[Iterable[Any]],
+        where: Optional[Sequence[str]],
+        params: Optional[Sequence[str]],
+        tables: Optional[Sequence[str]],
+        order_by: Optional[Sequence[str]],
     ) -> None: ...
     def clear_deferred_loading(self) -> None: ...
-    def add_deferred_loading(self, field_names: Tuple[str]) -> None: ...
-    def add_immediate_loading(self, field_names: Tuple[str]) -> None: ...
+    def add_deferred_loading(self, field_names: Iterable[str]) -> None: ...
+    def add_immediate_loading(self, field_names: Iterable[str]) -> None: ...
     def get_loaded_field_names(self) -> Dict[Type[Model], Set[str]]: ...
     def get_loaded_field_names_cb(
         self, target: Dict[Type[Model], Set[str]], model: Type[Model], fields: Set[Field]

django-stubs/utils/_os.pyi

@@ -1,4 +1,5 @@
 from os.path import abspath
+from pathlib import Path
 from typing import Any, Union
 
 abspathu = abspath
@@ -7,3 +8,4 @@ def upath(path: Any): ...
 def npath(path: Any): ...
 def safe_join(base: Union[bytes, str], *paths: Any) -> str: ...
 def symlinks_supported() -> Any: ...
+def to_path(value: Union[Path, str]) -> Path: ...

django-stubs/utils/deprecation.pyi

@@ -5,6 +5,7 @@ from django.http.response import HttpResponse
 
 class RemovedInDjango30Warning(PendingDeprecationWarning): ...
 class RemovedInDjango31Warning(PendingDeprecationWarning): ...
+class RemovedInDjango40Warning(PendingDeprecationWarning): ...
 class RemovedInNextVersionWarning(DeprecationWarning): ...
 
 class warn_about_renamed_method:

django-stubs/utils/http.pyi

@@ -25,6 +25,9 @@ def urlsafe_base64_decode(s: Union[bytes, str]) -> bytes: ...
 def parse_etags(etag_str: str) -> List[str]: ...
 def quote_etag(etag_str: str) -> str: ...
 def is_same_domain(host: str, pattern: str) -> bool: ...
+def url_has_allowed_host_and_scheme(
+    url: Optional[str], allowed_hosts: Optional[Union[str, Iterable[str]]], require_https: bool = ...
+) -> bool: ...
 def is_safe_url(
     url: Optional[str], allowed_hosts: Optional[Union[str, Iterable[str]]], require_https: bool = ...
 ) -> bool: ...

django-stubs/views/debug.pyi

@@ -63,4 +63,4 @@ class ExceptionReporter:
     ): ...
 
 def technical_404_response(request: HttpRequest, exception: Http404) -> HttpResponse: ...
-def default_urlconf(request: HttpRequest) -> HttpResponse: ...
+def default_urlconf(request: Optional[HttpResponse]) -> HttpResponse: ...

scripts/django_tests_settings.py

@@ -1,3 +1,5 @@
+import django
+
 SECRET_KEY = '1'
 SITE_ID = 1
 
@@ -41,7 +43,6 @@ test_modules = [
     'bulk_create',
     'cache',
     'check_framework',
-    'choices',
     'conditional_processing',
     'constraints',
     'contenttypes_tests',
@@ -219,6 +220,9 @@ test_modules = [
     'wsgi',
 ]
 
+if django.VERSION[0] == 2:
+    test_modules += ['choices']
+
 invalid_apps = {
     'import_error_package',
 }

scripts/enabled_test_modules.py

@@ -264,6 +264,8 @@ IGNORED_ERRORS = {
         'Incompatible types in assignment (expression has type "Type[Person',
         'Incompatible types in assignment (expression has type "FloatModel", variable has type',
         '"ImageFile" has no attribute "was_opened"',
+        'Incompatible type for "size" of "FloatModel" (got "object", expected "Union[float, int, str, Combinable]")',
+        'Incompatible type for "value" of "IntegerModel" (got "object", expected',
     ],
     'model_indexes': [
         'Argument "condition" to "Index" has incompatible type "str"; expected "Optional[Q]"'
@@ -291,6 +293,9 @@ IGNORED_ERRORS = {
     'model_options': [
         'expression has type "Dict[str, Type[Model]]", target has type "OrderedDict',
     ],
+    'model_enums': [
+        "'bool' is not a valid base class",
+    ],
     'multiple_database': [
         'Unexpected attribute "extra_arg" for model "Book"'
     ],
@@ -341,12 +346,14 @@ IGNORED_ERRORS = {
         '"Collection[Any]" has no attribute "explain"',
         "Cannot resolve keyword 'unknown_field' into field",
         'Incompatible type for lookup \'tag\': (got "str", expected "Union[Tag, int, None]")',
+        'No overload variant of "__getitem__" of "QuerySet" matches argument type "str"',
     ],
     'requests': [
         'Incompatible types in assignment (expression has type "Dict[str, str]", variable has type "QueryDict")'
     ],
     'responses': [
-        'Argument 1 to "TextIOWrapper" has incompatible type "HttpResponse"; expected "IO[bytes]"'
+        'Argument 1 to "TextIOWrapper" has incompatible type "HttpResponse"; expected "IO[bytes]"',
+        '"FileLike" has no attribute "closed"',
     ],
     'reverse_lookup': [
         "Cannot resolve keyword 'choice' into field"
@@ -424,6 +431,7 @@ IGNORED_ERRORS = {
         '"WSGIRequest" has no attribute "process_response_content"',
         'No overload variant of "join" matches argument types "str", "None"',
         'Argument 1 to "Archive" has incompatible type "None"; expected "str"',
+        'Argument 1 to "to_path" has incompatible type "int"; expected "Union[Path, str]"',
 
     ],
     'view_tests': [
@@ -431,10 +439,15 @@ IGNORED_ERRORS = {
         'Value of type "Optional[List[str]]" is not indexable',
         'ExceptionUser',
         'view_tests.tests.test_debug.User',
+        'Exception must be derived from BaseException',
+        "No binding for nonlocal 'tb_frames' found",
     ],
     'validation': [
         'has no attribute "name"',
     ],
+    'wsgi': [
+        '"HttpResponse" has no attribute "block_size"',
+    ],
 }
 
 

scripts/typecheck_tests.py

@@ -2,15 +2,23 @@ import itertools
 import shutil
 import subprocess
 import sys
+from argparse import ArgumentParser
 from collections import defaultdict
 from pathlib import Path
 from typing import Dict, List, Pattern, Union
 
+from git import Repo
+
 from scripts.enabled_test_modules import (
     EXTERNAL_MODULES, IGNORED_ERRORS, IGNORED_MODULES, MOCK_OBJECTS,
 )
 
+DJANGO_COMMIT_REFS = {
+    '2.2': 'e8b0903976077b951795938b260211214ed7fe41',
+    '3.0': '7ec5962638144cbf4c2e47ea7d8dc02d1ce44394'
+}
 PROJECT_DIRECTORY = Path(__file__).parent.parent
+DJANGO_SOURCE_DIRECTORY = PROJECT_DIRECTORY / 'django-sources'  # type: Path
 
 
 def get_unused_ignores(ignored_message_freq: Dict[str, Dict[Union[str, Pattern], int]]) -> List[str]:
@@ -67,11 +75,29 @@ def replace_with_clickable_location(error: str, abs_test_folder: Path) -> str:
     return error.replace(raw_path, clickable_location)
 
 
+def get_django_repo_object() -> Repo:
+    if not DJANGO_SOURCE_DIRECTORY.exists():
+        DJANGO_SOURCE_DIRECTORY.mkdir(exist_ok=True, parents=False)
+        return Repo.clone_from('https://github.com/django/django.git', DJANGO_SOURCE_DIRECTORY)
+    else:
+        repo = Repo(DJANGO_SOURCE_DIRECTORY)
+        return repo
+
+
 if __name__ == '__main__':
+    parser = ArgumentParser()
+    parser.add_argument('--django_version', choices=['2.2', '3.0'], required=True)
+    args = parser.parse_args()
+
+    commit_sha = DJANGO_COMMIT_REFS[args.django_version]
+    repo = get_django_repo_object()
+    if repo.head.commit.hexsha != commit_sha:
+        repo.git.fetch('origin')
+        repo.git.checkout(commit_sha)
+
     mypy_config_file = (PROJECT_DIRECTORY / 'scripts' / 'mypy.ini').absolute()
-    repo_directory = PROJECT_DIRECTORY / 'django-sources'
     mypy_cache_dir = Path(__file__).parent / '.mypy_cache'
-    tests_root = repo_directory / 'tests'
+    tests_root = DJANGO_SOURCE_DIRECTORY / 'tests'
     global_rc = 0
 
     try: