mirror of
https://github.com/davidhalter/parso.git
synced 2025-12-06 12:54:29 +08:00
Fix the token types.
This commit is contained in:
@@ -4,7 +4,8 @@ from parso.tokenize import group
|
||||
|
||||
|
||||
class PrefixToken(object):
|
||||
def __init__(self, value, start_pos):
|
||||
def __init__(self, typ, value, start_pos):
|
||||
self.type = typ
|
||||
self.value = value
|
||||
self.start_pos = start_pos
|
||||
|
||||
@@ -29,7 +30,7 @@ _types = {
|
||||
' ': 'spaces',
|
||||
'#': 'comment',
|
||||
'\\': 'backslash',
|
||||
'\f': 'form_feed',
|
||||
'\f': 'formfeed',
|
||||
'\n': 'newline',
|
||||
'\r': 'newline'
|
||||
}
|
||||
@@ -41,10 +42,10 @@ def split_prefix(prefix, start_pos):
|
||||
while start != len(prefix):
|
||||
match =_regex.match(prefix, start)
|
||||
value = match.group(0)
|
||||
yield PrefixToken(value, (line, column + start))
|
||||
typ = _types[value[0]]
|
||||
yield PrefixToken(typ, value, (line, column + start))
|
||||
|
||||
start = match.end(0)
|
||||
print(start)
|
||||
if '\n' in value:
|
||||
line += 1
|
||||
column = -start
|
||||
|
||||
@@ -22,3 +22,17 @@ def test_simple_prefix_splitting(string, tokens):
|
||||
assert leaf.type == 'endmarker'
|
||||
parsed_tokens = list(leaf._split_prefix())
|
||||
assert [t.value for t in parsed_tokens] == tokens
|
||||
|
||||
|
||||
@pytest.mark.parametrize(('string', 'types'), [
|
||||
('# ', ['comment']),
|
||||
('\r\n', ['newline']),
|
||||
('\f', ['formfeed']),
|
||||
('\\\n', ['backslash']),
|
||||
])
|
||||
def test_prefix_splitting_types(string, types):
|
||||
tree = parso.parse(string)
|
||||
leaf = tree.children[0]
|
||||
assert leaf.type == 'endmarker'
|
||||
parsed_tokens = list(leaf._split_prefix())
|
||||
assert [t.type for t in parsed_tokens] == types
|
||||
|
||||
Reference in New Issue
Block a user