VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-07 05:14:29 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Remove exact_op_types as a parameter, because it's not even used.

Browse Source
This commit is contained in:
Dave Halter
2017-05-26 11:49:48 -04:00
parent 0d93a2a2df
commit 33c228aa42
1 changed files with 6 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
parso/tokenize.py
View File

@@ -204,13 +204,13 @@ class TokenInfo(namedtuple('Token', ['type', 'string', 'start_pos', 'prefix'])):
return self.start_pos[0], self.start_pos[1] + len(self.string) return self.start_pos[0], self.start_pos[1] + len(self.string)
def source_tokens(source, exact_op_types=True): def source_tokens(source):
"""Generate tokens from a the source code (string).""" """Generate tokens from a the source code (string)."""
lines = splitlines(source, keepends=True) lines = splitlines(source, keepends=True)
return generate_tokens(lines, exact_op_types) return generate_tokens(lines)
def generate_tokens(lines, exact_op_types=True): def generate_tokens(lines):
""" """
A heavily modified Python standard library tokenizer. A heavily modified Python standard library tokenizer.
@@ -335,13 +335,9 @@ def generate_tokens(lines, exact_op_types=True):
try: try:
# This check is needed in any case to check if it's a valid # This check is needed in any case to check if it's a valid
# operator or just some random unicode character. # operator or just some random unicode character.
exact_type = opmap[token] typ = opmap[token]
except KeyError: except KeyError:
exact_type = typ = ERRORTOKEN typ = ERRORTOKEN
if exact_op_types:
typ = exact_type
else:
typ = OP
yield TokenInfo(typ, token, spos, prefix) yield TokenInfo(typ, token, spos, prefix)
if contstr: if contstr:
@@ -365,5 +361,5 @@ if __name__ == "__main__":
code = u(f.read()) code = u(f.read())
else: else:
code = u(sys.stdin.read()) code = u(sys.stdin.read())
for token in source_tokens(code, exact_op_types=True): for token in source_tokens(code):
print(token) print(token)