source_to_unicode -> python_bytes_to_unicode.

This commit is contained in:
Dave Halter
2017-08-15 20:07:24 +02:00
parent ab027885c7
commit 51f2de28c6
5 changed files with 10 additions and 10 deletions

View File

@@ -3,7 +3,7 @@ import os
from parso._compatibility import FileNotFoundError, is_pypy
from parso.pgen2.pgen import generate_grammar
from parso.utils import split_lines, source_to_unicode, parse_version_string
from parso.utils import split_lines, python_bytes_to_unicode, parse_version_string
from parso.python.diff import DiffParser
from parso.python.tokenize import tokenize_lines, tokenize
from parso.cache import parser_cache, load_module, save_module
@@ -86,7 +86,7 @@ class Grammar(object):
with open(path, 'rb') as f:
code = f.read()
code = source_to_unicode(code)
code = python_bytes_to_unicode(code)
lines = split_lines(code, keepends=True)
if diff_cache:

View File

@@ -406,10 +406,10 @@ if __name__ == "__main__":
else:
code = sys.stdin.read()
from parso.utils import source_to_unicode, parse_version_string
from parso.utils import python_bytes_to_unicode, parse_version_string
if isinstance(code, bytes):
code = source_to_unicode(code)
code = python_bytes_to_unicode(code)
for token in tokenize(code, parse_version_string()):
print(token)

View File

@@ -48,7 +48,7 @@ def split_lines(string, keepends=False):
return re.split('\n|\r\n', string)
def source_to_unicode(source, default_encoding='utf-8', errors='strict'):
def python_bytes_to_unicode(source, default_encoding='utf-8', errors='strict'):
"""
`errors` can be 'strict', 'replace' or 'ignore'.
"""

View File

@@ -8,7 +8,7 @@ import re
import parso
from parso._compatibility import total_ordering
from parso.utils import source_to_unicode
from parso.utils import python_bytes_to_unicode
@total_ordering
@@ -54,7 +54,7 @@ def test_normalizer_issue(normalizer_issue_case):
for i in issues]
with open(normalizer_issue_case.path, 'rb') as f:
code = source_to_unicode(f.read())
code = python_bytes_to_unicode(f.read())
desired = sort(collect_errors(code))

View File

@@ -1,6 +1,6 @@
from codecs import BOM_UTF8
from parso.utils import split_lines, source_to_unicode
from parso.utils import split_lines, python_bytes_to_unicode
import parso
@@ -20,12 +20,12 @@ def test_split_lines_keepends():
assert split_lines('\n', keepends=True) == ['\n', '']
def test_source_to_unicode_unicode_text():
def test_python_bytes_to_unicode_unicode_text():
source = (
b"# vim: fileencoding=utf-8\n"
b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n"
)
actual = source_to_unicode(source)
actual = python_bytes_to_unicode(source)
expected = source.decode('utf-8')
assert actual == expected