VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

tokenize corrections, add unicode literals, because they had been removed from Python 3.2 (reintroduced in 3.3)

Browse Source
This commit is contained in:
Dave Halter
2014-04-22 15:17:48 +02:00
parent b685101efb
commit a49c624154
1 changed files with 6 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
jedi/parser/tokenize.py
View File

@@ -163,9 +163,9 @@ plain_token = group(number, funny, string, name)
token = ignore + plain_token token = ignore + plain_token
# First (or only) line of ' or " string. # First (or only) line of ' or " string.
cont_str = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + cont_str = group(r"[bBuU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
group("'", r'\\\r?\n'), group("'", r'\\\r?\n'),
r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + r'[bBuU]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
group('"', r'\\\r?\n')) group('"', r'\\\r?\n'))
pseudo_extras = group(r'\\\r?\n', comment, triple) pseudo_extras = group(r'\\\r?\n', comment, triple)
pseudo_token = whitespace + group(pseudo_extras, number, funny, cont_str, name) pseudo_token = whitespace + group(pseudo_extras, number, funny, cont_str, name)
@@ -181,9 +181,11 @@ endprogs = {"'": _compile(single), '"': _compile(double),
"'''": single3prog, '"""': double3prog, "'''": single3prog, '"""': double3prog,
"r'''": single3prog, 'r"""': double3prog, "r'''": single3prog, 'r"""': double3prog,
"b'''": single3prog, 'b"""': double3prog, "b'''": single3prog, 'b"""': double3prog,
"u'''": single3prog, 'u"""': double3prog,
"br'''": single3prog, 'br"""': double3prog, "br'''": single3prog, 'br"""': double3prog,
"R'''": single3prog, 'R"""': double3prog, "R'''": single3prog, 'R"""': double3prog,
"B'''": single3prog, 'B"""': double3prog, "B'''": single3prog, 'B"""': double3prog,
"U'''": single3prog, 'U"""': double3prog,
"bR'''": single3prog, 'bR"""': double3prog, "bR'''": single3prog, 'bR"""': double3prog,
"Br'''": single3prog, 'Br"""': double3prog, "Br'''": single3prog, 'Br"""': double3prog,
"BR'''": single3prog, 'BR"""': double3prog, "BR'''": single3prog, 'BR"""': double3prog,
@@ -193,6 +195,7 @@ triple_quoted = {}
for t in ("'''", '"""', for t in ("'''", '"""',
"r'''", 'r"""', "R'''", 'R"""', "r'''", 'r"""', "R'''", 'R"""',
"b'''", 'b"""', "B'''", 'B"""', "b'''", 'b"""', "B'''", 'B"""',
"u'''", 'u"""', "U'''", 'U"""',
"br'''", 'br"""', "Br'''", 'Br"""', "br'''", 'br"""', "Br'''", 'Br"""',
"bR'''", 'bR"""', "BR'''", 'BR"""'): "bR'''", 'bR"""', "BR'''", 'BR"""'):
triple_quoted[t] = t triple_quoted[t] = t
@@ -200,6 +203,7 @@ single_quoted = {}
for t in ("'", '"', for t in ("'", '"',
"r'", 'r"', "R'", 'R"', "r'", 'r"', "R'", 'R"',
"b'", 'b"', "B'", 'B"', "b'", 'b"', "B'", 'B"',
"u'", 'u""', "U'", 'U"',
"br'", 'br"', "Br'", 'Br"', "br'", 'br"', "Br'", 'Br"',
"bR'", 'bR"', "BR'", 'BR"'): "bR'", 'bR"', "BR'", 'BR"'):
single_quoted[t] = t single_quoted[t] = t