use_exact_op_types -> exact_op_types.

This commit is contained in:
Dave Halter
2017-05-26 11:39:29 -04:00
parent b4259b2b4f
commit 8948c9053b
2 changed files with 6 additions and 6 deletions
+1 -1
View File
@@ -13,7 +13,7 @@ from parso import tokenize
class ParserGenerator(object): class ParserGenerator(object):
def __init__(self, bnf_text): def __init__(self, bnf_text):
self._bnf_text = bnf_text self._bnf_text = bnf_text
self.generator = tokenize.source_tokens(bnf_text, use_exact_op_types=False) self.generator = tokenize.source_tokens(bnf_text, exact_op_types=False)
self._gettoken() # Initialize lookahead self._gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self._parse() self.dfas, self.startsymbol = self._parse()
self.first = {} # map from symbol name to set of tokens self.first = {} # map from symbol name to set of tokens
+5 -5
View File
@@ -204,13 +204,13 @@ class TokenInfo(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])):
return self.start_pos[0], self.start_pos[1] + len(self.string) return self.start_pos[0], self.start_pos[1] + len(self.string)
def source_tokens(source, use_exact_op_types=True): def source_tokens(source, exact_op_types=True):
"""Generate tokens from a the source code (string).""" """Generate tokens from a the source code (string)."""
lines = splitlines(source, keepends=True) lines = splitlines(source, keepends=True)
return generate_tokens(lines, use_exact_op_types) return generate_tokens(lines, exact_op_types)
def generate_tokens(lines, use_exact_op_types=True): def generate_tokens(lines, exact_op_types=True):
""" """
A heavily modified Python standard library tokenizer. A heavily modified Python standard library tokenizer.
@@ -338,7 +338,7 @@ def generate_tokens(lines, use_exact_op_types=True):
exact_type = opmap[token] exact_type = opmap[token]
except KeyError: except KeyError:
exact_type = typ = ERRORTOKEN exact_type = typ = ERRORTOKEN
if use_exact_op_types: if exact_op_types:
typ = exact_type typ = exact_type
else: else:
typ = OP typ = OP
@@ -365,5 +365,5 @@ if __name__ == "__main__":
code = u(f.read()) code = u(f.read())
else: else:
code = u(sys.stdin.read()) code = u(sys.stdin.read())
for token in source_tokens(code, use_exact_op_types=True): for token in source_tokens(code, exact_op_types=True):
print(token) print(token)