mirror of
https://github.com/davidhalter/parso.git
synced 2025-12-06 21:04:29 +08:00
source_to_unicode -> python_bytes_to_unicode.
This commit is contained in:
@@ -3,7 +3,7 @@ import os
|
||||
|
||||
from parso._compatibility import FileNotFoundError, is_pypy
|
||||
from parso.pgen2.pgen import generate_grammar
|
||||
from parso.utils import split_lines, source_to_unicode, parse_version_string
|
||||
from parso.utils import split_lines, python_bytes_to_unicode, parse_version_string
|
||||
from parso.python.diff import DiffParser
|
||||
from parso.python.tokenize import tokenize_lines, tokenize
|
||||
from parso.cache import parser_cache, load_module, save_module
|
||||
@@ -86,7 +86,7 @@ class Grammar(object):
|
||||
with open(path, 'rb') as f:
|
||||
code = f.read()
|
||||
|
||||
code = source_to_unicode(code)
|
||||
code = python_bytes_to_unicode(code)
|
||||
|
||||
lines = split_lines(code, keepends=True)
|
||||
if diff_cache:
|
||||
|
||||
@@ -406,10 +406,10 @@ if __name__ == "__main__":
|
||||
else:
|
||||
code = sys.stdin.read()
|
||||
|
||||
from parso.utils import source_to_unicode, parse_version_string
|
||||
from parso.utils import python_bytes_to_unicode, parse_version_string
|
||||
|
||||
if isinstance(code, bytes):
|
||||
code = source_to_unicode(code)
|
||||
code = python_bytes_to_unicode(code)
|
||||
|
||||
for token in tokenize(code, parse_version_string()):
|
||||
print(token)
|
||||
|
||||
@@ -48,7 +48,7 @@ def split_lines(string, keepends=False):
|
||||
return re.split('\n|\r\n', string)
|
||||
|
||||
|
||||
def source_to_unicode(source, default_encoding='utf-8', errors='strict'):
|
||||
def python_bytes_to_unicode(source, default_encoding='utf-8', errors='strict'):
|
||||
"""
|
||||
`errors` can be 'strict', 'replace' or 'ignore'.
|
||||
"""
|
||||
|
||||
@@ -8,7 +8,7 @@ import re
|
||||
|
||||
import parso
|
||||
from parso._compatibility import total_ordering
|
||||
from parso.utils import source_to_unicode
|
||||
from parso.utils import python_bytes_to_unicode
|
||||
|
||||
|
||||
@total_ordering
|
||||
@@ -54,7 +54,7 @@ def test_normalizer_issue(normalizer_issue_case):
|
||||
for i in issues]
|
||||
|
||||
with open(normalizer_issue_case.path, 'rb') as f:
|
||||
code = source_to_unicode(f.read())
|
||||
code = python_bytes_to_unicode(f.read())
|
||||
|
||||
desired = sort(collect_errors(code))
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
from codecs import BOM_UTF8
|
||||
|
||||
from parso.utils import split_lines, source_to_unicode
|
||||
from parso.utils import split_lines, python_bytes_to_unicode
|
||||
import parso
|
||||
|
||||
|
||||
@@ -20,12 +20,12 @@ def test_split_lines_keepends():
|
||||
assert split_lines('\n', keepends=True) == ['\n', '']
|
||||
|
||||
|
||||
def test_source_to_unicode_unicode_text():
|
||||
def test_python_bytes_to_unicode_unicode_text():
|
||||
source = (
|
||||
b"# vim: fileencoding=utf-8\n"
|
||||
b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n"
|
||||
)
|
||||
actual = source_to_unicode(source)
|
||||
actual = python_bytes_to_unicode(source)
|
||||
expected = source.decode('utf-8')
|
||||
assert actual == expected
|
||||
|
||||
|
||||
Reference in New Issue
Block a user