VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2026-04-14 03:39:49 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

source_to_unicode -> python_bytes_to_unicode.

Browse Source
This commit is contained in:
Dave Halter
2017-08-15 20:07:24 +02:00
parent ab027885c7
commit 51f2de28c6
5 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
parso/grammar.py
View File

@@ -3,7 +3,7 @@ import os
from parso._compatibility import FileNotFoundError, is_pypy
from parso.pgen2.pgen import generate_grammar
from parso.utils import split_lines, source_to_unicode, parse_version_string
from parso.utils import split_lines, python_bytes_to_unicode, parse_version_string
from parso.python.diff import DiffParser
from parso.python.tokenize import tokenize_lines, tokenize
from parso.cache import parser_cache, load_module, save_module
@@ -86,7 +86,7 @@ class Grammar(object):
with open(path, 'rb') as f:
code = f.read()
code = source_to_unicode(code)
code = python_bytes_to_unicode(code)
lines = split_lines(code, keepends=True)
if diff_cache:

4
parso/python/tokenize.py
View File

@@ -406,10 +406,10 @@ if __name__ == "__main__":
else:
code = sys.stdin.read()
from parso.utils import source_to_unicode, parse_version_string
from parso.utils import python_bytes_to_unicode, parse_version_string
if isinstance(code, bytes):
code = source_to_unicode(code)
code = python_bytes_to_unicode(code)
for token in tokenize(code, parse_version_string()):
print(token)

2
parso/utils.py
View File

@@ -48,7 +48,7 @@ def split_lines(string, keepends=False):
return re.split('\n|\r\n', string)
def source_to_unicode(source, default_encoding='utf-8', errors='strict'):
def python_bytes_to_unicode(source, default_encoding='utf-8', errors='strict'):
"""
`errors` can be 'strict', 'replace' or 'ignore'.
"""