From 3e4b2f41cd0dc60326cafbdeacc6e0be91ca271d Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Tue, 30 May 2017 02:05:45 +0200 Subject: [PATCH] Fix the token types. --- parso/python/prefix.py | 9 +++++---- test/test_prefix.py | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/parso/python/prefix.py b/parso/python/prefix.py index f6d3142..38547e1 100644 --- a/parso/python/prefix.py +++ b/parso/python/prefix.py @@ -4,7 +4,8 @@ from parso.tokenize import group class PrefixToken(object): - def __init__(self, value, start_pos): + def __init__(self, typ, value, start_pos): + self.type = typ self.value = value self.start_pos = start_pos @@ -29,7 +30,7 @@ _types = { ' ': 'spaces', '#': 'comment', '\\': 'backslash', - '\f': 'form_feed', + '\f': 'formfeed', '\n': 'newline', '\r': 'newline' } @@ -41,10 +42,10 @@ def split_prefix(prefix, start_pos): while start != len(prefix): match =_regex.match(prefix, start) value = match.group(0) - yield PrefixToken(value, (line, column + start)) + typ = _types[value[0]] + yield PrefixToken(typ, value, (line, column + start)) start = match.end(0) - print(start) if '\n' in value: line += 1 column = -start diff --git a/test/test_prefix.py b/test/test_prefix.py index c0883b5..f0e040a 100644 --- a/test/test_prefix.py +++ b/test/test_prefix.py @@ -22,3 +22,17 @@ def test_simple_prefix_splitting(string, tokens): assert leaf.type == 'endmarker' parsed_tokens = list(leaf._split_prefix()) assert [t.value for t in parsed_tokens] == tokens + + +@pytest.mark.parametrize(('string', 'types'), [ + ('# ', ['comment']), + ('\r\n', ['newline']), + ('\f', ['formfeed']), + ('\\\n', ['backslash']), +]) +def test_prefix_splitting_types(string, types): + tree = parso.parse(string) + leaf = tree.children[0] + assert leaf.type == 'endmarker' + parsed_tokens = list(leaf._split_prefix()) + assert [t.type for t in parsed_tokens] == types