stdlib/3/ast: add visit_* methods to NodeVisitor (#3796)

NodeVisitor recurses over an AST tree. When encountering a node, it
checks if a method called `visit_{node.__class__.__name__}` exists, and
calls it if so, otherwise calls the generic visitor.

Add the possible methods to NodeVisitor. This is not exactly correct,
since the methods don't *actually* exist on NodeVisitor, e.g.
`NodeVisitor().visit_Module(...)` doesn't work. But it's nice for
subclasses to know which methods they can override and which type they
should have.

stdlib/3/ast.pyi

@@ -19,11 +19,128 @@ else:
 class NodeVisitor:
     def visit(self, node: AST) -> Any: ...
     def generic_visit(self, node: AST) -> Any: ...
-    if sys.version_info >= (3, 8):
+    def visit_Module(self, node: Module) -> Any: ...
+    def visit_Interactive(self, node: Interactive) -> Any: ...
+    def visit_Expression(self, node: Expression) -> Any: ...
+    def visit_Suite(self, node: Suite) -> Any: ...
+    def visit_FunctionDef(self, node: FunctionDef) -> Any: ...
+    def visit_AsyncFunctionDef(self, node: AsyncFunctionDef) -> Any: ...
+    def visit_ClassDef(self, node: ClassDef) -> Any: ...
+    def visit_Return(self, node: Return) -> Any: ...
+    def visit_Delete(self, node: Delete) -> Any: ...
+    def visit_Assign(self, node: Assign) -> Any: ...
+    def visit_AugAssign(self, node: AugAssign) -> Any: ...
+    if sys.version_info >= (3, 6):
+        def visit_AnnAssign(self, node: AnnAssign) -> Any: ...
+    def visit_For(self, node: For) -> Any: ...
+    def visit_AsyncFor(self, node: AsyncFor) -> Any: ...
+    def visit_While(self, node: While) -> Any: ...
+    def visit_If(self, node: If) -> Any: ...
+    def visit_With(self, node: With) -> Any: ...
+    def visit_AsyncWith(self, node: AsyncWith) -> Any: ...
+    def visit_Raise(self, node: Raise) -> Any: ...
+    def visit_Try(self, node: Try) -> Any: ...
+    def visit_Assert(self, node: Assert) -> Any: ...
+    def visit_Import(self, node: Import) -> Any: ...
+    def visit_ImportFrom(self, node: ImportFrom) -> Any: ...
+    def visit_Global(self, node: Global) -> Any: ...
+    def visit_Nonlocal(self, node: Nonlocal) -> Any: ...
+    def visit_Expr(self, node: Expr) -> Any: ...
+    def visit_Pass(self, node: Pass) -> Any: ...
+    def visit_Break(self, node: Break) -> Any: ...
+    def visit_Continue(self, node: Continue) -> Any: ...
+    def visit_Slice(self, node: Slice) -> Any: ...
+    def visit_ExtSlice(self, node: ExtSlice) -> Any: ...
+    def visit_Index(self, node: Index) -> Any: ...
+    def visit_BoolOp(self, node: BoolOp) -> Any: ...
+    def visit_BinOp(self, node: BinOp) -> Any: ...
+    def visit_UnaryOp(self, node: UnaryOp) -> Any: ...
+    def visit_Lambda(self, node: Lambda) -> Any: ...
+    def visit_IfExp(self, node: IfExp) -> Any: ...
+    def visit_Dict(self, node: Dict) -> Any: ...
+    def visit_Set(self, node: Set) -> Any: ...
+    def visit_ListComp(self, node: ListComp) -> Any: ...
+    def visit_SetComp(self, node: SetComp) -> Any: ...
+    def visit_DictComp(self, node: DictComp) -> Any: ...
+    def visit_GeneratorExp(self, node: GeneratorExp) -> Any: ...
+    def visit_Await(self, node: Await) -> Any: ...
+    def visit_Yield(self, node: Yield) -> Any: ...
+    def visit_YieldFrom(self, node: YieldFrom) -> Any: ...
+    def visit_Compare(self, node: Compare) -> Any: ...
+    def visit_Call(self, node: Call) -> Any: ...
+    if sys.version_info >= (3, 6):
+        def visit_FormattedValue(self, node: FormattedValue) -> Any: ...
+        def visit_JoinedStr(self, node: JoinedStr) -> Any: ...
+    if sys.version_info < (3, 8):
+        def visit_Num(self, node: Num) -> Any: ...
+        def visit_Str(self, node: Str) -> Any: ...
+        def visit_Bytes(self, node: Bytes) -> Any: ...
+        def visit_NameConstant(self, node: NameConstant) -> Any: ...
+        def visit_Ellipsis(self, node: Ellipsis) -> Any: ...
+    else:
+        def visit_Num(self, node: Constant) -> Any: ...
+        def visit_Str(self, node: Constant) -> Any: ...
+        def visit_Bytes(self, node: Constant) -> Any: ...
+        def visit_NameConstant(self, node: Constant) -> Any: ...
+        def visit_Ellipsis(self, node: Constant) -> Any: ...
+    if sys.version_info >= (3, 6):
         def visit_Constant(self, node: Constant) -> Any: ...
+    if sys.version_info >= (3, 8):
+        def visit_NamedExpr(self, node: NamedExpr) -> Any: ...
+    def visit_Attribute(self, node: Attribute) -> Any: ...
+    def visit_Subscript(self, node: Subscript) -> Any: ...
+    def visit_Starred(self, node: Starred) -> Any: ...
+    def visit_Name(self, node: Name) -> Any: ...
+    def visit_List(self, node: List) -> Any: ...
+    def visit_Tuple(self, node: Tuple) -> Any: ...
+    def visit_AugLoad(self, node: AugLoad) -> Any: ...
+    def visit_AugStore(self, node: AugStore) -> Any: ...
+    def visit_Del(self, node: Del) -> Any: ...
+    def visit_Load(self, node: Load) -> Any: ...
+    def visit_Param(self, node: Param) -> Any: ...
+    def visit_Store(self, node: Store) -> Any: ...
+    def visit_And(self, node: And) -> Any: ...
+    def visit_Or(self, node: Or) -> Any: ...
+    def visit_Add(self, node: Add) -> Any: ...
+    def visit_BitAnd(self, node: BitAnd) -> Any: ...
+    def visit_BitOr(self, node: BitOr) -> Any: ...
+    def visit_BitXor(self, node: BitXor) -> Any: ...
+    def visit_Div(self, node: Div) -> Any: ...
+    def visit_FloorDiv(self, node: FloorDiv) -> Any: ...
+    def visit_LShift(self, node: LShift) -> Any: ...
+    def visit_Mod(self, node: Mod) -> Any: ...
+    def visit_Mult(self, node: Mult) -> Any: ...
+    def visit_MatMult(self, node: MatMult) -> Any: ...
+    def visit_Pow(self, node: Pow) -> Any: ...
+    def visit_RShift(self, node: RShift) -> Any: ...
+    def visit_Sub(self, node: Sub) -> Any: ...
+    def visit_Invert(self, node: Invert) -> Any: ...
+    def visit_Not(self, node: Not) -> Any: ...
+    def visit_UAdd(self, node: UAdd) -> Any: ...
+    def visit_USub(self, node: USub) -> Any: ...
+    def visit_Eq(self, node: Eq) -> Any: ...
+    def visit_Gt(self, node: Gt) -> Any: ...
+    def visit_GtE(self, node: GtE) -> Any: ...
+    def visit_In(self, node: In) -> Any: ...
+    def visit_Is(self, node: Is) -> Any: ...
+    def visit_IsNot(self, node: IsNot) -> Any: ...
+    def visit_Lt(self, node: Lt) -> Any: ...
+    def visit_LtE(self, node: LtE) -> Any: ...
+    def visit_NotEq(self, node: NotEq) -> Any: ...
+    def visit_NotIn(self, node: NotIn) -> Any: ...
+    def visit_comprehension(self, node: comprehension) -> Any: ...
+    def visit_ExceptHandler(self, node: ExceptHandler) -> Any: ...
+    def visit_arguments(self, node: arguments) -> Any: ...
+    def visit_arg(self, node: arg) -> Any: ...
+    def visit_keyword(self, node: keyword) -> Any: ...
+    def visit_alias(self, node: alias) -> Any: ...
+    def visit_withitem(self, node: withitem) -> Any: ...
 
 class NodeTransformer(NodeVisitor):
     def generic_visit(self, node: AST) -> Optional[AST]: ...
+    # TODO: Override the visit_* methods with better return types.
+    #       The usual return type is Optional[AST], but Iterable[AST]
+    #       is also allowed in some cases -- this needs to be mapped.
 
 _T = TypeVar("_T", bound=AST)
 

tests/stubtest_whitelists/py36.txt

@@ -1,3 +1,7 @@
+ast.NodeVisitor.visit_AnnAssign
+ast.NodeVisitor.visit_Constant
+ast.NodeVisitor.visit_FormattedValue
+ast.NodeVisitor.visit_JoinedStr
 asyncio.Future.__init__
 asyncio.Task._wakeup
 asyncio.constants.SENDFILE_FALLBACK_READBUFFER_SIZE

tests/stubtest_whitelists/py37.txt

@@ -2,6 +2,10 @@ _bisect.bisect
 _bisect.insort
 _tracemalloc._get_object_traceback
 _tracemalloc.start
+ast.NodeVisitor.visit_AnnAssign
+ast.NodeVisitor.visit_Constant
+ast.NodeVisitor.visit_FormattedValue
+ast.NodeVisitor.visit_JoinedStr
 asyncio.AbstractEventLoop.create_unix_server
 asyncio.AbstractEventLoop.sock_sendfile
 asyncio.Future.__init__

tests/stubtest_whitelists/py38.txt

@@ -5,6 +5,10 @@ _thread._ExceptHookArgs
 _tracemalloc._get_object_traceback
 _tracemalloc.start
 _weakref.getweakrefcount
+ast.NodeVisitor.visit_AnnAssign
+ast.NodeVisitor.visit_FormattedValue
+ast.NodeVisitor.visit_JoinedStr
+ast.NodeVisitor.visit_NamedExpr
 asyncio.AbstractEventLoop.create_unix_server
 asyncio.AbstractEventLoop.sock_sendfile
 asyncio.Future.__init__

tests/stubtest_whitelists/py3_common.txt

@@ -15,6 +15,106 @@ abc.abstractstaticmethod
 aifc.open
 aifc.openfp
 argparse.Namespace.__getattr__
+ast.NodeVisitor.visit_Add
+ast.NodeVisitor.visit_And
+ast.NodeVisitor.visit_Assert
+ast.NodeVisitor.visit_Assign
+ast.NodeVisitor.visit_AsyncFor
+ast.NodeVisitor.visit_AsyncFunctionDef
+ast.NodeVisitor.visit_AsyncWith
+ast.NodeVisitor.visit_Attribute
+ast.NodeVisitor.visit_AugAssign
+ast.NodeVisitor.visit_AugLoad
+ast.NodeVisitor.visit_AugStore
+ast.NodeVisitor.visit_Await
+ast.NodeVisitor.visit_BinOp
+ast.NodeVisitor.visit_BitAnd
+ast.NodeVisitor.visit_BitOr
+ast.NodeVisitor.visit_BitXor
+ast.NodeVisitor.visit_BoolOp
+ast.NodeVisitor.visit_Break
+ast.NodeVisitor.visit_Bytes
+ast.NodeVisitor.visit_Call
+ast.NodeVisitor.visit_ClassDef
+ast.NodeVisitor.visit_Compare
+ast.NodeVisitor.visit_Continue
+ast.NodeVisitor.visit_Del
+ast.NodeVisitor.visit_Delete
+ast.NodeVisitor.visit_Dict
+ast.NodeVisitor.visit_DictComp
+ast.NodeVisitor.visit_Div
+ast.NodeVisitor.visit_Ellipsis
+ast.NodeVisitor.visit_Eq
+ast.NodeVisitor.visit_ExceptHandler
+ast.NodeVisitor.visit_Expr
+ast.NodeVisitor.visit_Expression
+ast.NodeVisitor.visit_ExtSlice
+ast.NodeVisitor.visit_FloorDiv
+ast.NodeVisitor.visit_For
+ast.NodeVisitor.visit_FunctionDef
+ast.NodeVisitor.visit_GeneratorExp
+ast.NodeVisitor.visit_Global
+ast.NodeVisitor.visit_Gt
+ast.NodeVisitor.visit_GtE
+ast.NodeVisitor.visit_If
+ast.NodeVisitor.visit_IfExp
+ast.NodeVisitor.visit_Import
+ast.NodeVisitor.visit_ImportFrom
+ast.NodeVisitor.visit_In
+ast.NodeVisitor.visit_Index
+ast.NodeVisitor.visit_Interactive
+ast.NodeVisitor.visit_Invert
+ast.NodeVisitor.visit_Is
+ast.NodeVisitor.visit_IsNot
+ast.NodeVisitor.visit_LShift
+ast.NodeVisitor.visit_Lambda
+ast.NodeVisitor.visit_List
+ast.NodeVisitor.visit_ListComp
+ast.NodeVisitor.visit_Load
+ast.NodeVisitor.visit_Lt
+ast.NodeVisitor.visit_LtE
+ast.NodeVisitor.visit_MatMult
+ast.NodeVisitor.visit_Mod
+ast.NodeVisitor.visit_Module
+ast.NodeVisitor.visit_Mult
+ast.NodeVisitor.visit_Name
+ast.NodeVisitor.visit_NameConstant
+ast.NodeVisitor.visit_Nonlocal
+ast.NodeVisitor.visit_Not
+ast.NodeVisitor.visit_NotEq
+ast.NodeVisitor.visit_NotIn
+ast.NodeVisitor.visit_Num
+ast.NodeVisitor.visit_Or
+ast.NodeVisitor.visit_Param
+ast.NodeVisitor.visit_Pass
+ast.NodeVisitor.visit_Pow
+ast.NodeVisitor.visit_RShift
+ast.NodeVisitor.visit_Raise
+ast.NodeVisitor.visit_Return
+ast.NodeVisitor.visit_Set
+ast.NodeVisitor.visit_SetComp
+ast.NodeVisitor.visit_Slice
+ast.NodeVisitor.visit_Starred
+ast.NodeVisitor.visit_Store
+ast.NodeVisitor.visit_Str
+ast.NodeVisitor.visit_Sub
+ast.NodeVisitor.visit_Subscript
+ast.NodeVisitor.visit_Suite
+ast.NodeVisitor.visit_Try
+ast.NodeVisitor.visit_Tuple
+ast.NodeVisitor.visit_UAdd
+ast.NodeVisitor.visit_USub
+ast.NodeVisitor.visit_UnaryOp
+ast.NodeVisitor.visit_While
+ast.NodeVisitor.visit_With
+ast.NodeVisitor.visit_Yield
+ast.NodeVisitor.visit_YieldFrom
+ast.NodeVisitor.visit_alias
+ast.NodeVisitor.visit_arg
+ast.NodeVisitor.visit_arguments
+ast.NodeVisitor.visit_comprehension
+ast.NodeVisitor.visit_keyword
+ast.NodeVisitor.visit_withitem
 asyncio.AbstractEventLoop.connect_accepted_socket
 asyncio.AbstractEventLoop.create_unix_connection
 asyncio.BaseChildWatcher