VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-01-21 02:22:23 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

use_exact_op_types -> exact_op_types.

Browse Source
This commit is contained in:
Dave Halter
2017-05-26 11:39:29 -04:00
parent b4259b2b4f
commit 8948c9053b
2 changed files with 6 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
parso/pgen2/pgen.py
View File

@@ -13,7 +13,7 @@ from parso import tokenize
class ParserGenerator(object):
def __init__(self, bnf_text):
self._bnf_text = bnf_text
self.generator = tokenize.source_tokens(bnf_text, use_exact_op_types=False)
self.generator = tokenize.source_tokens(bnf_text, exact_op_types=False)
self._gettoken() # Initialize lookahead
self.dfas, self.startsymbol = self._parse()
self.first = {} # map from symbol name to set of tokens

10
parso/tokenize.py
View File

@@ -204,13 +204,13 @@ class TokenInfo(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])):
return self.start_pos[0], self.start_pos[1] + len(self.string)
def source_tokens(source, use_exact_op_types=True):
def source_tokens(source, exact_op_types=True):
"""Generate tokens from a the source code (string)."""
lines = splitlines(source, keepends=True)
return generate_tokens(lines, use_exact_op_types)
return generate_tokens(lines, exact_op_types)
def generate_tokens(lines, use_exact_op_types=True):
def generate_tokens(lines, exact_op_types=True):
"""
A heavily modified Python standard library tokenizer.
@@ -338,7 +338,7 @@ def generate_tokens(lines, use_exact_op_types=True):
exact_type = opmap[token]
except KeyError:
exact_type = typ = ERRORTOKEN
if use_exact_op_types:
if exact_op_types:
typ = exact_type
else:
typ = OP
@@ -365,5 +365,5 @@ if __name__ == "__main__":
code = u(f.read())
else:
code = u(sys.stdin.read())
for token in source_tokens(code, use_exact_op_types=True):
for token in source_tokens(code, exact_op_types=True):
print(token)