Specific return types for values and values list (#53)

* Instead of using Literal types, overload QuerySet.values_list in the plugin. Fixes #43.

- Add a couple of extra type checks that Django makes:
  1) 'flat' and 'named' can't be used together.
  2) 'flat' is not valid when values_list is called with more than one field.

* Determine better row types for values_list/values based on fields specified.

- In the case of values_list, we use a Row type with either a single primitive, Tuple, or NamedTuple.
- In the case of values, we use a TypedDict.
- In both cases, Any is used as a fallback for individual fields if those fields cannot be resolved.

A couple other fixes I made along the way:
- Don't create reverse relation for ForeignKeys with related_name='+'
- Don't skip creating other related managers in AddRelatedManagers if a dynamic value is encountered
  for related_name parameter, or if the type cannot be determined.

* Fix for TypedDict so that they are considered anonymous.

* Clean up some comments.

* Implement making TypedDict anonymous in a way that doesn't crash sometimes.

* Fix flake8 errors.

* Remove even uglier hack about making TypedDict anonymous.

* Address review comments. Write a few better comments inside tests.

* Fix crash when running with mypyc ("interpreted classes cannot inherit from compiled") due to the way I extended TypedDictType.

- Implemented the hack in another way that works on mypyc.
- Added a couple extra tests of accessing 'id' / 'pk' via values_list.

* Fix flake8 errors.

* Support annotation expressions (use type Any) for TypedDicts row types returned by values_list.

- Bonus points: handle values_list gracefully (use type Any) where Tuples are returned
  where some of the fields arguments are not string literals.

mypy_django_plugin/helpers.py

@@ -1,14 +1,16 @@
 import typing
-from typing import Dict, Optional
+from collections import OrderedDict
+from typing import Dict, Optional, cast
 
-from mypy.checker import TypeChecker
+from mypy.checker import TypeChecker, gen_unique_name
+from mypy.mro import calculate_mro
 from mypy.nodes import (
     AssignmentStmt, ClassDef, Expression, ImportedName, Lvalue, MypyFile, NameExpr, SymbolNode, TypeInfo,
-)
+    SymbolTable, SymbolTableNode, Block, GDEF, MDEF, Var)
 from mypy.plugin import FunctionContext, MethodContext
 from mypy.types import (
     AnyType, Instance, NoneTyp, Type, TypeOfAny, TypeVarType, UnionType,
-)
+    TupleType, TypedDictType)
 
 MODEL_CLASS_FULLNAME = 'django.db.models.base.Model'
 FIELD_FULLNAME = 'django.db.models.fields.Field'
@@ -211,7 +213,7 @@ def extract_field_setter_type(tp: Instance) -> Optional[Type]:
     return None
 
 
-def extract_field_getter_type(tp: Instance) -> Optional[Type]:
+def extract_field_getter_type(tp: Type) -> Optional[Type]:
     if not isinstance(tp, Instance):
         return None
     if tp.type.has_base(FIELD_FULLNAME):
@@ -235,6 +237,10 @@ def get_fields_metadata(model: TypeInfo) -> Dict[str, typing.Any]:
     return get_django_metadata(model).setdefault('fields', {})
 
 
+def get_lookups_metadata(model: TypeInfo) -> Dict[str, typing.Any]:
+    return get_django_metadata(model).setdefault('lookups', {})
+
+
 def extract_explicit_set_type_of_model_primary_key(model: TypeInfo) -> Optional[Type]:
     """
     If field with primary_key=True is set on the model, extract its __set__ type.
@@ -296,3 +302,84 @@ def get_assigned_value_for_class(type_info: TypeInfo, name: str) -> Optional[Exp
         if isinstance(lvalue, NameExpr) and lvalue.name == name:
             return rvalue
     return None
+
+
+def is_field_nullable(model: TypeInfo, field_name: str) -> bool:
+    return get_fields_metadata(model).get(field_name, {}).get('null', False)
+
+
+def is_foreign_key(t: Type) -> bool:
+    if not isinstance(t, Instance):
+        return False
+    return has_any_of_bases(t.type, (FOREIGN_KEY_FULLNAME, ONETOONE_FIELD_FULLNAME))
+
+
+def build_class_with_annotated_fields(api: TypeChecker, base: Type, fields: 'OrderedDict[str, Type]',
+                                      name: str) -> Instance:
+    """Build an Instance with `name` that contains the specified `fields` as attributes and extends `base`."""
+    # Credit: This code is largely copied/modified from TypeChecker.intersect_instance_callable and
+    # NamedTupleAnalyzer.build_namedtuple_typeinfo
+
+    cur_module = cast(MypyFile, api.scope.stack[0])
+    gen_name = gen_unique_name(name, cur_module.names)
+
+    cdef = ClassDef(name, Block([]))
+    cdef.fullname = cur_module.fullname() + '.' + gen_name
+    info = TypeInfo(SymbolTable(), cdef, cur_module.fullname())
+    cdef.info = info
+    info.bases = [base]
+
+    def add_field(var: Var, is_initialized_in_class: bool = False,
+                  is_property: bool = False) -> None:
+        var.info = info
+        var.is_initialized_in_class = is_initialized_in_class
+        var.is_property = is_property
+        var._fullname = '%s.%s' % (info.fullname(), var.name())
+        info.names[var.name()] = SymbolTableNode(MDEF, var)
+
+    vars = [Var(item, typ) for item, typ in fields.items()]
+    for var in vars:
+        add_field(var, is_property=True)
+
+    calculate_mro(info)
+    info.calculate_metaclass_type()
+
+    cur_module.names[gen_name] = SymbolTableNode(GDEF, info, plugin_generated=True)
+    return Instance(info, [])
+
+
+def make_named_tuple(api: TypeChecker, fields: 'OrderedDict[str, Type]', name: str) -> Type:
+    if not fields:
+        # No fields specified, so fallback to a subclass of NamedTuple that allows
+        # __getattr__ / __setattr__ for any attribute name.
+        fallback = api.named_generic_type('django._NamedTupleAnyAttr', [])
+    else:
+        fallback = build_class_with_annotated_fields(
+            api=api,
+            base=api.named_generic_type('typing.NamedTuple', []),
+            fields=fields,
+            name=name
+        )
+    return TupleType(list(fields.values()), fallback=fallback)
+
+
+def make_typeddict(api: TypeChecker, fields: 'OrderedDict[str, Type]', required_keys: typing.Set[str]) -> Type:
+    object_type = api.named_generic_type('mypy_extensions._TypedDict', [])
+    typed_dict_type = TypedDictType(fields, required_keys=required_keys, fallback=object_type)
+    return typed_dict_type
+
+
+def make_tuple(api: TypeChecker, fields: typing.List[Type]) -> Type:
+    implicit_any = AnyType(TypeOfAny.special_form)
+    fallback = api.named_generic_type('builtins.tuple', [implicit_any])
+    return TupleType(fields, fallback=fallback)
+
+
+def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is_nullable: bool) -> Type:
+    node = type_info.get(private_field_name).node
+    if isinstance(node, Var):
+        descriptor_type = node.type
+        if is_nullable:
+            descriptor_type = make_optional(descriptor_type)
+        return descriptor_type
+    return AnyType(TypeOfAny.unannotated)