mirror of
https://github.com/davidhalter/parso.git
synced 2025-12-07 21:34:32 +08:00
Fix the token types.
This commit is contained in:
@@ -4,7 +4,8 @@ from parso.tokenize import group
|
|||||||
|
|
||||||
|
|
||||||
class PrefixToken(object):
|
class PrefixToken(object):
|
||||||
def __init__(self, value, start_pos):
|
def __init__(self, typ, value, start_pos):
|
||||||
|
self.type = typ
|
||||||
self.value = value
|
self.value = value
|
||||||
self.start_pos = start_pos
|
self.start_pos = start_pos
|
||||||
|
|
||||||
@@ -29,7 +30,7 @@ _types = {
|
|||||||
' ': 'spaces',
|
' ': 'spaces',
|
||||||
'#': 'comment',
|
'#': 'comment',
|
||||||
'\\': 'backslash',
|
'\\': 'backslash',
|
||||||
'\f': 'form_feed',
|
'\f': 'formfeed',
|
||||||
'\n': 'newline',
|
'\n': 'newline',
|
||||||
'\r': 'newline'
|
'\r': 'newline'
|
||||||
}
|
}
|
||||||
@@ -41,10 +42,10 @@ def split_prefix(prefix, start_pos):
|
|||||||
while start != len(prefix):
|
while start != len(prefix):
|
||||||
match =_regex.match(prefix, start)
|
match =_regex.match(prefix, start)
|
||||||
value = match.group(0)
|
value = match.group(0)
|
||||||
yield PrefixToken(value, (line, column + start))
|
typ = _types[value[0]]
|
||||||
|
yield PrefixToken(typ, value, (line, column + start))
|
||||||
|
|
||||||
start = match.end(0)
|
start = match.end(0)
|
||||||
print(start)
|
|
||||||
if '\n' in value:
|
if '\n' in value:
|
||||||
line += 1
|
line += 1
|
||||||
column = -start
|
column = -start
|
||||||
|
|||||||
@@ -22,3 +22,17 @@ def test_simple_prefix_splitting(string, tokens):
|
|||||||
assert leaf.type == 'endmarker'
|
assert leaf.type == 'endmarker'
|
||||||
parsed_tokens = list(leaf._split_prefix())
|
parsed_tokens = list(leaf._split_prefix())
|
||||||
assert [t.value for t in parsed_tokens] == tokens
|
assert [t.value for t in parsed_tokens] == tokens
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(('string', 'types'), [
|
||||||
|
('# ', ['comment']),
|
||||||
|
('\r\n', ['newline']),
|
||||||
|
('\f', ['formfeed']),
|
||||||
|
('\\\n', ['backslash']),
|
||||||
|
])
|
||||||
|
def test_prefix_splitting_types(string, types):
|
||||||
|
tree = parso.parse(string)
|
||||||
|
leaf = tree.children[0]
|
||||||
|
assert leaf.type == 'endmarker'
|
||||||
|
parsed_tokens = list(leaf._split_prefix())
|
||||||
|
assert [t.type for t in parsed_tokens] == types
|
||||||
|
|||||||
Reference in New Issue
Block a user