mirror of
https://github.com/davidhalter/parso.git
synced 2026-08-19 06:18:04 +08:00
use_exact_op_types -> exact_op_types.
This commit is contained in:
+1
-1
@@ -13,7 +13,7 @@ from parso import tokenize
|
|||||||
class ParserGenerator(object):
|
class ParserGenerator(object):
|
||||||
def __init__(self, bnf_text):
|
def __init__(self, bnf_text):
|
||||||
self._bnf_text = bnf_text
|
self._bnf_text = bnf_text
|
||||||
self.generator = tokenize.source_tokens(bnf_text, use_exact_op_types=False)
|
self.generator = tokenize.source_tokens(bnf_text, exact_op_types=False)
|
||||||
self._gettoken() # Initialize lookahead
|
self._gettoken() # Initialize lookahead
|
||||||
self.dfas, self.startsymbol = self._parse()
|
self.dfas, self.startsymbol = self._parse()
|
||||||
self.first = {} # map from symbol name to set of tokens
|
self.first = {} # map from symbol name to set of tokens
|
||||||
|
|||||||
+5
-5
@@ -204,13 +204,13 @@ class TokenInfo(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])):
|
|||||||
return self.start_pos[0], self.start_pos[1] + len(self.string)
|
return self.start_pos[0], self.start_pos[1] + len(self.string)
|
||||||
|
|
||||||
|
|
||||||
def source_tokens(source, use_exact_op_types=True):
|
def source_tokens(source, exact_op_types=True):
|
||||||
"""Generate tokens from a the source code (string)."""
|
"""Generate tokens from a the source code (string)."""
|
||||||
lines = splitlines(source, keepends=True)
|
lines = splitlines(source, keepends=True)
|
||||||
return generate_tokens(lines, use_exact_op_types)
|
return generate_tokens(lines, exact_op_types)
|
||||||
|
|
||||||
|
|
||||||
def generate_tokens(lines, use_exact_op_types=True):
|
def generate_tokens(lines, exact_op_types=True):
|
||||||
"""
|
"""
|
||||||
A heavily modified Python standard library tokenizer.
|
A heavily modified Python standard library tokenizer.
|
||||||
|
|
||||||
@@ -338,7 +338,7 @@ def generate_tokens(lines, use_exact_op_types=True):
|
|||||||
exact_type = opmap[token]
|
exact_type = opmap[token]
|
||||||
except KeyError:
|
except KeyError:
|
||||||
exact_type = typ = ERRORTOKEN
|
exact_type = typ = ERRORTOKEN
|
||||||
if use_exact_op_types:
|
if exact_op_types:
|
||||||
typ = exact_type
|
typ = exact_type
|
||||||
else:
|
else:
|
||||||
typ = OP
|
typ = OP
|
||||||
@@ -365,5 +365,5 @@ if __name__ == "__main__":
|
|||||||
code = u(f.read())
|
code = u(f.read())
|
||||||
else:
|
else:
|
||||||
code = u(sys.stdin.read())
|
code = u(sys.stdin.read())
|
||||||
for token in source_tokens(code, use_exact_op_types=True):
|
for token in source_tokens(code, exact_op_types=True):
|
||||||
print(token)
|
print(token)
|
||||||
|
|||||||
Reference in New Issue
Block a user