sre_parse: various fixes (#3758)

* sre_parse: NIC is stub implementation detail

* sre_parse: add missing arg to Tokenizer.getuntil

* sre_parse: fix types for parse

* sre_parse: fix types for SubPattern.__init__

* sre_parse: fix arg name for fix_flags

* sre_parse: update whitelists

* sre_parse: fix some availability for py35

stdlib/3/sre_parse.pyi

@@ -5,7 +5,7 @@ from typing import (
     Optional, Pattern as _Pattern, Tuple, Union
 )
 import sys
-from sre_constants import _NamedIntConstant as NIC, error as _Error
+from sre_constants import _NamedIntConstant as _NIC, error as _Error
 
 SPECIAL_CHARS: str
 REPEAT_CHARS: str
@@ -14,12 +14,13 @@ OCTDIGITS: FrozenSet[str]
 HEXDIGITS: FrozenSet[str]
 ASCIILETTERS: FrozenSet[str]
 WHITESPACE: FrozenSet[str]
-ESCAPES: Dict[str, Tuple[NIC, int]]
-CATEGORIES: Dict[str, Union[Tuple[NIC, NIC], Tuple[NIC, List[Tuple[NIC, NIC]]]]]
+ESCAPES: Dict[str, Tuple[_NIC, int]]
+CATEGORIES: Dict[str, Union[Tuple[_NIC, _NIC], Tuple[_NIC, List[Tuple[_NIC, _NIC]]]]]
 FLAGS: Dict[str, int]
-GLOBAL_FLAGS: int
+if sys.version_info >= (3, 6):
+    GLOBAL_FLAGS: int
 
-class Verbose(Exception): ...
+    class Verbose(Exception): ...
 
 class _State:
     flags: int
@@ -42,10 +43,10 @@ else:
 
 _OpSubpatternType = Tuple[Optional[int], int, int, SubPattern]
 _OpGroupRefExistsType = Tuple[int, SubPattern, SubPattern]
-_OpInType = List[Tuple[NIC, int]]
+_OpInType = List[Tuple[_NIC, int]]
 _OpBranchType = Tuple[None, List[SubPattern]]
 _AvType = Union[_OpInType, _OpBranchType, Iterable[SubPattern], _OpGroupRefExistsType, _OpSubpatternType]
-_CodeType = Tuple[NIC, _AvType]
+_CodeType = Tuple[_NIC, _AvType]
 
 
 class SubPattern:
@@ -54,10 +55,10 @@ class SubPattern:
 
     if sys.version_info >= (3, 8):
         state: State
-        def __init__(self, state: State, data: List[_CodeType] = ...) -> None: ...
+        def __init__(self, state: State, data: Optional[List[_CodeType]] = ...) -> None: ...
     else:
         pattern: Pattern
-        def __init__(self, pattern: Pattern, data: List[_CodeType] = ...) -> None: ...
+        def __init__(self, pattern: Pattern, data: Optional[List[_CodeType]] = ...) -> None: ...
 
     def dump(self, level: int = ...) -> None: ...
     def __len__(self) -> int: ...
@@ -79,19 +80,23 @@ class Tokenizer:
     def match(self, char: str) -> bool: ...
     def get(self) -> Optional[str]: ...
     def getwhile(self, n: int, charset: Iterable[str]) -> str: ...
-    def getuntil(self, terminator: str) -> str: ...
-    @property
-    def pos(self) -> int: ...
+    if sys.version_info >= (3, 8):
+        def getuntil(self, terminator: str, name: str) -> str: ...
+    else:
+        def getuntil(self, terminator: str) -> str: ...
+    if sys.version_info >= (3, 6):
+        @property
+        def pos(self) -> int: ...
     def tell(self) -> int: ...
     def seek(self, index: int) -> None: ...
     def error(self, msg: str, offset: int = ...) -> _Error: ...
 
-def fix_flags(src: Union[str, bytes], flag: int) -> int: ...
+def fix_flags(src: Union[str, bytes], flags: int) -> int: ...
 _TemplateType = Tuple[List[Tuple[int, int]], List[str]]
 if sys.version_info >= (3, 8):
-    def parse(str: str, flags: int = ..., state: State = ...) -> SubPattern: ...
+    def parse(str: str, flags: int = ..., state: Optional[State] = ...) -> SubPattern: ...
     def parse_template(source: str, state: _Pattern[Any]) -> _TemplateType: ...
 else:
-    def parse(str: str, flags: int = ..., pattern: Pattern = ...) -> SubPattern: ...
+    def parse(str: str, flags: int = ..., pattern: Optional[Pattern] = ...) -> SubPattern: ...
     def parse_template(source: str, pattern: _Pattern[Any]) -> _TemplateType: ...
 def expand_template(template: _TemplateType, match: Match[Any]) -> str: ...

tests/stubtest_whitelists/py35.txt

@@ -72,9 +72,6 @@ smtpd.SMTPChannel.__init__
 smtpd.SMTPServer.__init__
 socket.NETLINK_CRYPTO
 sre_compile.dis
-sre_parse.GLOBAL_FLAGS
-sre_parse.Tokenizer.pos
-sre_parse.Verbose
 subprocess.check_output
 tracemalloc.Filter.__init__
 typing.AbstractSet.isdisjoint

tests/stubtest_whitelists/py38.txt

@@ -174,7 +174,6 @@ signal.sigtimedwait
 signal.sigwaitinfo
 smtplib.SMTP.sendmail
 sre_constants.RANGE_IGNORE
-sre_parse.Tokenizer.getuntil
 ssl.MemoryBIO.read
 ssl.MemoryBIO.write
 ssl.RAND_bytes

tests/stubtest_whitelists/py3_common.txt

@@ -703,10 +703,6 @@ sqlite3.dbapi2.version_info
 sqlite3.register_adapters_and_converters
 sqlite3.version_info
 sre_constants.error.__init__
-sre_parse.NIC
-sre_parse.SubPattern.__init__
-sre_parse.fix_flags
-sre_parse.parse
 ssl.PROTOCOL_SSLv2
 ssl.PROTOCOL_SSLv3
 ssl.RAND_add