From 18dc92f85f7dce7beeeae3831cb14f446f992e99 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Mon, 28 Apr 2014 18:33:40 +0200 Subject: [PATCH] removed a few old/unnecessary tokenize definitions --- jedi/parser/tokenize.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 39f3491b..5d6161d3 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -119,7 +119,6 @@ def maybe(*choices): # number literals. whitespace = r'[ \f\t]*' comment = r'#[^\r\n]*' -ignore = whitespace + any(r'\\\r?\n' + whitespace) + maybe(comment) name = r'\w+' hex_number = r'0[xX][0-9a-fA-F]+' @@ -144,8 +143,6 @@ single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' triple = group("[bB]?[rR]?'''", '[bB]?[rR]?"""') # Single-line ' or " string. -string = group(r"[bB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", - r'[bB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') # Because of leftmost-then-longest match semantics, be sure to put the # longest operators first (e.g., if = came before ==, == would get @@ -159,9 +156,6 @@ bracket = '[][(){}]' special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]') funny = group(operator, bracket, special) -plain_token = group(number, funny, string, name) -token = ignore + plain_token - # First (or only) line of ' or " string. cont_str = group(r"[bBuU]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + group("'", r'\\\r?\n'), @@ -175,8 +169,8 @@ def _compile(expr): return re.compile(expr, re.UNICODE) -tokenprog, pseudoprog, single3prog, double3prog = map( - _compile, (token, pseudo_token, single3, double3)) +pseudoprog, single3prog, double3prog = map( + _compile, (pseudo_token, single3, double3)) endprogs = {"'": _compile(single), '"': _compile(double), "'''": single3prog, '"""': double3prog, "r'''": single3prog, 'r"""': double3prog,