VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-08-12 19:11:34 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor f-string support

Browse Source
This commit is contained in:
Dave Halter
2019-01-23 10:58:36 +01:00
parent 60e4591837
commit 5fbc207892
1 changed files with 25 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files
parso/python/tokenize.py
+25 -52
View File
@@ -284,26 +284,20 @@ class FStringNode(object):
return (self.parentheses_count - self.format_spec_count) > 0 return (self.parentheses_count - self.format_spec_count) > 0
def _check_fstring_ending(fstring_stack, token, from_start=False): def _close_fstring_if_necessary(fstring_stack, string, start_pos, additional_prefix):
fstring_end = float('inf') for fstring_stack_index, node in enumerate(fstring_stack):
fstring_index = None if string.startswith(node.quote):
for i, node in enumerate(fstring_stack): token = PythonToken(
if from_start: FSTRING_END,
if token.startswith(node.quote): node.quote,
fstring_index = i start_pos,
fstring_end = len(node.quote) prefix=additional_prefix,
else: )
continue additional_prefix = ''
else: assert not node.previous_lines
try: del fstring_stack[fstring_stack_index:]
end = token.index(node.quote) return token, '', len(node.quote)
except ValueError: return None, additional_prefix, 0
pass
else:
if fstring_index is None or end < fstring_end:
fstring_index = i
fstring_end = end
return fstring_index, fstring_end
def _find_fstring_string(endpats, fstring_stack, line, lnum, pos): def _find_fstring_string(endpats, fstring_stack, line, lnum, pos):
@@ -425,6 +419,8 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
tos = fstring_stack[-1] tos = fstring_stack[-1]
if not tos.is_in_expr(): if not tos.is_in_expr():
string, pos = _find_fstring_string(endpats, fstring_stack, line, lnum, pos) string, pos = _find_fstring_string(endpats, fstring_stack, line, lnum, pos)
if pos == max:
break
if string: if string:
yield PythonToken( yield PythonToken(
FSTRING_STRING, string, FSTRING_STRING, string,
@@ -436,22 +432,16 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
tos.previous_lines = '' tos.previous_lines = ''
continue continue
if pos == max:
break
rest = line[pos:] rest = line[pos:]
fstring_index, end = _check_fstring_ending(fstring_stack, rest, from_start=True) fstring_end_token, additional_prefix, quote_length = _close_fstring_if_necessary(
fstring_stack,
if fstring_index is not None: rest,
yield PythonToken( (lnum, pos),
FSTRING_END, additional_prefix,
fstring_stack[fstring_index].quote, )
(lnum, pos), pos += quote_length
prefix=additional_prefix, if fstring_end_token is not None:
) yield fstring_end_token
additional_prefix = ''
del fstring_stack[fstring_index:]
pos += end
continue continue
pseudomatch = pseudo_token.match(line, pos) pseudomatch = pseudo_token.match(line, pos)
@@ -498,23 +488,6 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
for t in dedent_if_necessary(indent_start): for t in dedent_if_necessary(indent_start):
yield t yield t
if fstring_stack:
fstring_index, end = _check_fstring_ending(fstring_stack, token)
if fstring_index is not None:
if end != 0:
yield PythonToken(ERRORTOKEN, token[:end], spos, prefix)
prefix = ''
yield PythonToken(
FSTRING_END,
fstring_stack[fstring_index].quote,
(lnum, spos[1] + 1),
prefix=prefix
)
del fstring_stack[fstring_index:]
pos -= len(token) - end
continue
if (initial in numchars or # ordinary number if (initial in numchars or # ordinary number
(initial == '.' and token != '.' and token != '...')): (initial == '.' and token != '.' and token != '...')):
yield PythonToken(NUMBER, token, spos, prefix) yield PythonToken(NUMBER, token, spos, prefix)