VimPlug/jedi
1
0
Fork 1
mirror of https://github.com/davidhalter/jedi.git synced 2025-12-06 05:54:25 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

changed _compatibility.utf8 -> 'u' and removed a lot of the issues with the now enforced unicode source input of the parser

Browse Source
This commit is contained in:
Dave Halter
2014-02-23 11:29:00 +01:00
parent 5478e50f8b
commit c5fcebde82
10 changed files with 41 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
jedi/_compatibility.py
View File

@@ -90,11 +90,11 @@ except NameError:
unicode = str
if is_py3:
utf8 = lambda s: s
u = lambda s: s
else:
utf8 = lambda s: s.decode('utf-8')
u = lambda s: s.decode('utf-8')
utf8.__doc__ = """
u.__doc__ = """
Decode a raw string into unicode object. Do nothing in Python 3.
"""

4
jedi/api/interpreter.py
View File

@@ -3,6 +3,7 @@ import re
from jedi._compatibility import builtins
from jedi import debug
from jedi.common import source_to_unicode
from jedi.cache import underscore_memoization
from jedi.evaluate import compiled
from jedi.evaluate.compiled.fake import get_module
@@ -68,7 +69,8 @@ class LazyName(helpers.FakeName):
if path.endswith('.py'):
# cut the `c` from `.pyc`
with open(path) as f:
mod = FastParser(f.read(), path[:-1]).module
source = source_to_unicode(f.read())
mod = FastParser(source, path[:-1]).module
if not parser_path:
return mod
found = self._evaluator.eval_call_path(iter(parser_path), mod, None)

4
jedi/evaluate/compiled/fake.py
View File

@@ -7,7 +7,7 @@ mixing in Python code, the autocompletion should work much better for builtins.
import os
import inspect
from jedi._compatibility import is_py3, builtins
from jedi._compatibility import is_py3, builtins, unicode
from jedi.parser import Parser
from jedi.parser import token as token_pr
from jedi.parser.representation import Class
@@ -31,7 +31,7 @@ def _load_faked_module(module):
except IOError:
modules[module_name] = None
return
module = Parser(source, module_name).module
module = Parser(unicode(source), module_name).module
modules[module_name] = module
if module_name == 'builtins' and not is_py3:

4
jedi/parser/fast.py
View File

@@ -5,7 +5,7 @@ finished (and still not working as I want), I won't document it any further.
"""
import re
from jedi._compatibility import use_metaclass
from jedi._compatibility import use_metaclass, unicode
from jedi import settings
from jedi import common
from jedi.parser import Parser
@@ -275,7 +275,7 @@ class FastParser(use_metaclass(CachedFastParser)):
def _parse(self, code):
""" :type code: str """
def empty_parser():
new, temp = self._get_parser('', '', 0, [], False)
new, temp = self._get_parser(unicode(''), unicode(''), 0, [], False)
return new
parts = self._split_parts(code)

6
jedi/parser/token.py
View File

@@ -8,7 +8,7 @@ found that a flat object with slots is the best.
from inspect import cleandoc
from ast import literal_eval
from jedi._compatibility import utf8, unicode
from jedi._compatibility import u, unicode
class Token(object):
@@ -37,7 +37,7 @@ class Token(object):
4
>>> Token.from_tuple((6, 5, (4, 3)))
<Token: (6, 5, (4, 3))>
>>> unicode(Token(1, utf8("😷"), 1 ,1)) + "p" == utf8("😷p")
>>> unicode(Token(1, u("😷"), 1 ,1)) + "p" == u("😷p")
True
"""
__slots__ = ("_token_type", "_token", "_start_pos_line", "_start_pos_col")
@@ -166,4 +166,4 @@ class TokenDocstring(TokenNoCompat):
def as_string(self):
"""Returns a literal cleaned version of the token"""
return cleandoc(literal_eval(self.token))
return unicode(cleandoc(literal_eval(self.token)))

15
jedi/parser/tokenize.py
View File

@@ -12,7 +12,8 @@ from __future__ import absolute_import
import string
import re
from io import StringIO
from token import *
from token import (tok_name, N_TOKENS, ENDMARKER, STRING, NUMBER, NAME, OP,
ERRORTOKEN, NEWLINE)
import collections
cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
@@ -23,9 +24,8 @@ namechars = string.ascii_letters + '_'
COMMENT = N_TOKENS
tok_name[COMMENT] = 'COMMENT'
ENCODING = N_TOKENS + 2
ENCODING = N_TOKENS + 1
tok_name[ENCODING] = 'ENCODING'
N_TOKENS += 3
class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end')):
@@ -153,7 +153,6 @@ def generate_tokens(readline, line_offset=0):
Modified to not care about dedents.
"""
lnum = line_offset
continued = False
numchars = '0123456789'
contstr = ''
contline = None
@@ -161,7 +160,7 @@ def generate_tokens(readline, line_offset=0):
line = readline() # readline returns empty if it's finished. See StringIO
if not line:
if contstr:
yield TokenInfo(ERRORTOKEN, contstr, strstart, (lnum, pos))
yield TokenInfo(ERRORTOKEN, contstr, contstr_start, (lnum, pos))
break
lnum += 1
@@ -171,7 +170,7 @@ def generate_tokens(readline, line_offset=0):
endmatch = endprog.match(line)
if endmatch:
pos = end = endmatch.end(0)
yield TokenInfo(STRING, contstr + line[:end], strstart, (lnum, end))
yield TokenInfo(STRING, contstr + line[:end], contstr_start, (lnum, end))
contstr = ''
contline = None
else:
@@ -207,7 +206,7 @@ def generate_tokens(readline, line_offset=0):
token = line[start:pos]
yield TokenInfo(STRING, token, spos, (lnum, pos))
else:
strstart = (lnum, start) # multiple lines
contstr_start = (lnum, start) # multiple lines
contstr = line[start:]
contline = line
break
@@ -215,7 +214,7 @@ def generate_tokens(readline, line_offset=0):
token[:2] in single_quoted or \
token[:3] in single_quoted:
if token[-1] == '\n': # continued string
strstart = (lnum, start)
contstr_start = lnum, start
endprog = (endprogs[initial] or endprogs[token[1]] or
endprogs[token[2]])
contstr = line[start:]

9
jedi/parser/user_context.py
View File

@@ -3,6 +3,7 @@ import os
from jedi import cache
from jedi.parser import tokenize
from jedi._compatibility import u
from jedi.parser.fast import FastParser
from jedi.parser import representation
from jedi import debug
@@ -70,7 +71,7 @@ class UserContext(object):
for token_type, tok, start, end in gen:
if is_first:
if start != (1, 0): # whitespace is not a path
return '', start_cursor
return u(''), start_cursor
is_first = False
# print 'tok', token_type, tok, force_point
@@ -167,14 +168,14 @@ class UserContext(object):
self._line_cache = self.source.splitlines()
if self.source:
if self.source[-1] == '\n':
self._line_cache.append('')
self._line_cache.append(u(''))
else: # ''.splitlines() == []
self._line_cache = ['']
self._line_cache = [u('')]
if line_nr == 0:
# This is a fix for the zeroth line. We need a newline there, for
# the backwards parser.
return ''
return u('')
if line_nr < 0:
raise StopIteration()
try:

2
scripts/wx_check.py
View File

@@ -37,6 +37,7 @@ def process_memory():
uri = 'http://svn.wxwidgets.org/viewvc/wx/wxPython/trunk/src/gtk/_core.py?revision=74740&content-type=text%2Fplain&view=co'
wx_core = urllib2.urlopen(uri).read()
wx_core = wx_core[:1]
def run():
@@ -44,6 +45,7 @@ def run():
print('Process Memory before: %skB' % process_memory())
# After this the module should be cached.
# Need to invent a path so that it's really cached.
print type(wx_core), wx_core
jedi.Script(wx_core, path='foobar.py').completions()
gc.collect() # make sure that it's all fair and the gc did its job.

20
test/test_api/test_unicode.py
View File

@@ -3,7 +3,7 @@
All character set and unicode related tests.
"""
from jedi import Script
from jedi._compatibility import utf8, unicode
from jedi._compatibility import u, unicode
def test_unicode_script():
@@ -13,12 +13,12 @@ def test_unicode_script():
assert len(completions)
assert type(completions[0].description) is unicode
s = utf8("author='öä'; author")
s = u("author='öä'; author")
completions = Script(s).completions()
x = completions[0].description
assert type(x) is unicode
s = utf8("#-*- coding: iso-8859-1 -*-\nauthor='öä'; author")
s = u("#-*- coding: iso-8859-1 -*-\nauthor='öä'; author")
s = s.encode('latin-1')
completions = Script(s).completions()
assert type(completions[0].description) is unicode
@@ -26,12 +26,12 @@ def test_unicode_script():
def test_unicode_attribute():
""" github jedi-vim issue #94 """
s1 = utf8('#-*- coding: utf-8 -*-\nclass Person():\n'
' name = "e"\n\nPerson().name.')
s1 = u('#-*- coding: utf-8 -*-\nclass Person():\n'
' name = "e"\n\nPerson().name.')
completions1 = Script(s1).completions()
assert 'strip' in [c.name for c in completions1]
s2 = utf8('#-*- coding: utf-8 -*-\nclass Person():\n'
' name = "é"\n\nPerson().name.')
s2 = u('#-*- coding: utf-8 -*-\nclass Person():\n'
' name = "é"\n\nPerson().name.')
completions2 = Script(s2).completions()
assert 'strip' in [c.name for c in completions2]
@@ -39,9 +39,9 @@ def test_unicode_attribute():
def test_multibyte_script():
""" `jedi.Script` must accept multi-byte string source. """
try:
code = unicode("import datetime; datetime.d")
comment = utf8("# multi-byte comment あいうえおä")
s = (unicode('%s\n%s') % (code, comment)).encode('utf-8')
code = u("import datetime; datetime.d")
comment = u("# multi-byte comment あいうえおä")
s = (u('%s\n%s') % (code, comment)).encode('utf-8')
except NameError:
pass # python 3 has no unicode method
else:

7
test/test_evaluate/test_absolute_import.py
View File

@@ -3,6 +3,7 @@ Tests ``from __future__ import absolute_import`` (only important for
Python 2.X)
"""
import jedi
from jedi._compatibility import u
from jedi.parser import Parser
from .. import helpers
@@ -11,7 +12,7 @@ def test_explicit_absolute_imports():
"""
Detect modules with ``from __future__ import absolute_import``.
"""
parser = Parser("from __future__ import absolute_import", "test.py")
parser = Parser(u("from __future__ import absolute_import"), "test.py")
assert parser.module.has_explicit_absolute_import
@@ -19,7 +20,7 @@ def test_no_explicit_absolute_imports():
"""
Detect modules without ``from __future__ import absolute_import``.
"""
parser = Parser("1", "test.py")
parser = Parser(u("1"), "test.py")
assert not parser.module.has_explicit_absolute_import
@@ -28,7 +29,7 @@ def test_dont_break_imports_without_namespaces():
The code checking for ``from __future__ import absolute_import`` shouldn't
assume that all imports have non-``None`` namespaces.
"""
src = "from __future__ import absolute_import\nimport xyzzy"
src = u("from __future__ import absolute_import\nimport xyzzy")
parser = Parser(src, "test.py")
assert parser.module.has_explicit_absolute_import