From fe69989fbc082e22a993e0252887cb7de170ff4d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 21 Jan 2019 00:33:02 +0100 Subject: [PATCH] Add a comment from the Python3.7 code base --- parso/python/tokenize.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/parso/python/tokenize.py b/parso/python/tokenize.py index ff7311f..da7560f 100644 --- a/parso/python/tokenize.py +++ b/parso/python/tokenize.py @@ -544,6 +544,17 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)): contstr = line[start:] contline = line break + + # Check up to the first 3 chars of the token to see if + # they're in the single_quoted set. If so, they start + # a string. + # We're using the first 3, because we're looking for + # "rb'" (for example) at the start of the token. If + # we switch to longer prefixes, this needs to be + # adjusted. + # Note that initial == token[:1]. + # Also note that single quote checking must come after + # triple quote checking (above). elif initial in single_quoted or \ token[:2] in single_quoted or \ token[:3] in single_quoted: