VimPlug/parso
1
0
Fork 0
mirror of https://github.com/davidhalter/parso.git synced 2025-12-07 21:34:32 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

source_to_unicode -> python_bytes_to_unicode.

Browse Source
This commit is contained in:
Dave Halter
2017-08-15 20:07:24 +02:00
parent ab027885c7
commit 51f2de28c6
5 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
parso/grammar.py
View File

@@ -3,7 +3,7 @@ import os
from parso._compatibility import FileNotFoundError, is_pypy from parso._compatibility import FileNotFoundError, is_pypy
from parso.pgen2.pgen import generate_grammar from parso.pgen2.pgen import generate_grammar
from parso.utils import split_lines, source_to_unicode, parse_version_string from parso.utils import split_lines, python_bytes_to_unicode, parse_version_string
from parso.python.diff import DiffParser from parso.python.diff import DiffParser
from parso.python.tokenize import tokenize_lines, tokenize from parso.python.tokenize import tokenize_lines, tokenize
from parso.cache import parser_cache, load_module, save_module from parso.cache import parser_cache, load_module, save_module
@@ -86,7 +86,7 @@ class Grammar(object):
with open(path, 'rb') as f: with open(path, 'rb') as f:
code = f.read() code = f.read()
code = source_to_unicode(code) code = python_bytes_to_unicode(code)
lines = split_lines(code, keepends=True) lines = split_lines(code, keepends=True)
if diff_cache: if diff_cache:

4
parso/python/tokenize.py
View File

@@ -406,10 +406,10 @@ if __name__ == "__main__":
else: else:
code = sys.stdin.read() code = sys.stdin.read()
from parso.utils import source_to_unicode, parse_version_string from parso.utils import python_bytes_to_unicode, parse_version_string
if isinstance(code, bytes): if isinstance(code, bytes):
code = source_to_unicode(code) code = python_bytes_to_unicode(code)
for token in tokenize(code, parse_version_string()): for token in tokenize(code, parse_version_string()):
print(token) print(token)

2
parso/utils.py
View File

@@ -48,7 +48,7 @@ def split_lines(string, keepends=False):
return re.split('\n|\r\n', string) return re.split('\n|\r\n', string)
def source_to_unicode(source, default_encoding='utf-8', errors='strict'): def python_bytes_to_unicode(source, default_encoding='utf-8', errors='strict'):
""" """
`errors` can be 'strict', 'replace' or 'ignore'. `errors` can be 'strict', 'replace' or 'ignore'.
""" """

4
test/test_normalizer_issues_files.py
View File

@@ -8,7 +8,7 @@ import re
import parso import parso
from parso._compatibility import total_ordering from parso._compatibility import total_ordering
from parso.utils import source_to_unicode from parso.utils import python_bytes_to_unicode
@total_ordering @total_ordering
@@ -54,7 +54,7 @@ def test_normalizer_issue(normalizer_issue_case):
for i in issues] for i in issues]
with open(normalizer_issue_case.path, 'rb') as f: with open(normalizer_issue_case.path, 'rb') as f:
code = source_to_unicode(f.read()) code = python_bytes_to_unicode(f.read())
desired = sort(collect_errors(code)) desired = sort(collect_errors(code))

6
test/test_utils.py
View File

@@ -1,6 +1,6 @@
from codecs import BOM_UTF8 from codecs import BOM_UTF8
from parso.utils import split_lines, source_to_unicode from parso.utils import split_lines, python_bytes_to_unicode
import parso import parso
@@ -20,12 +20,12 @@ def test_split_lines_keepends():
assert split_lines('\n', keepends=True) == ['\n', ''] assert split_lines('\n', keepends=True) == ['\n', '']
def test_source_to_unicode_unicode_text(): def test_python_bytes_to_unicode_unicode_text():
source = ( source = (
b"# vim: fileencoding=utf-8\n" b"# vim: fileencoding=utf-8\n"
b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n" b"# \xe3\x81\x82\xe3\x81\x84\xe3\x81\x86\xe3\x81\x88\xe3\x81\x8a\n"
) )
actual = source_to_unicode(source) actual = python_bytes_to_unicode(source)
expected = source.decode('utf-8') expected = source.decode('utf-8')
assert actual == expected assert actual == expected