Fix the token types.

This commit is contained in:
Dave Halter
2017-05-30 02:05:45 +02:00
parent 6640308d15
commit 3e4b2f41cd
2 changed files with 19 additions and 4 deletions

View File

@@ -4,7 +4,8 @@ from parso.tokenize import group
class PrefixToken(object):
def __init__(self, value, start_pos):
def __init__(self, typ, value, start_pos):
self.type = typ
self.value = value
self.start_pos = start_pos
@@ -29,7 +30,7 @@ _types = {
' ': 'spaces',
'#': 'comment',
'\\': 'backslash',
'\f': 'form_feed',
'\f': 'formfeed',
'\n': 'newline',
'\r': 'newline'
}
@@ -41,10 +42,10 @@ def split_prefix(prefix, start_pos):
while start != len(prefix):
match =_regex.match(prefix, start)
value = match.group(0)
yield PrefixToken(value, (line, column + start))
typ = _types[value[0]]
yield PrefixToken(typ, value, (line, column + start))
start = match.end(0)
print(start)
if '\n' in value:
line += 1
column = -start

View File

@@ -22,3 +22,17 @@ def test_simple_prefix_splitting(string, tokens):
assert leaf.type == 'endmarker'
parsed_tokens = list(leaf._split_prefix())
assert [t.value for t in parsed_tokens] == tokens
@pytest.mark.parametrize(('string', 'types'), [
('# ', ['comment']),
('\r\n', ['newline']),
('\f', ['formfeed']),
('\\\n', ['backslash']),
])
def test_prefix_splitting_types(string, types):
tree = parso.parse(string)
leaf = tree.children[0]
assert leaf.type == 'endmarker'
parsed_tokens = list(leaf._split_prefix())
assert [t.type for t in parsed_tokens] == types