From e980c8987b2f5b6ec8857f2f291b5648bb488c9f Mon Sep 17 00:00:00 2001
From: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Date: Tue, 4 Jul 2017 19:17:39 -0700
Subject: [PATCH] tokenize: add generate_tokens in py3 (#1449)

Fixes #1433

This is undocumented but somebody is asking for it to be included.
---
 stdlib/3/tokenize.pyi | 1 +
 1 file changed, 1 insertion(+)

diff --git a/stdlib/3/tokenize.pyi b/stdlib/3/tokenize.pyi
index 2e9518223..bf1339f0c 100644
--- a/stdlib/3/tokenize.pyi
+++ b/stdlib/3/tokenize.pyi
@@ -39,6 +39,7 @@ class Untokenizer:
 def untokenize(iterable: Iterable[_Token]) -> Any: ...
 def detect_encoding(readline: Callable[[], bytes]) -> Tuple[str, Sequence[bytes]]: ...
 def tokenize(readline: Callable[[], bytes]) -> Generator[TokenInfo, None, None]: ...
+def generate_tokens(readline: Callable[[], str]) -> Generator[TokenInfo, None, None]: ...
 
 def open(filename: Union[str, bytes, int]) -> TextIO: ...