From 39bf9f426be9a2ba38a57e3494686269aa3b9469 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Wed, 18 Feb 2015 17:32:34 +0100 Subject: [PATCH] Handle backslash escaping. --- jedi/parser/fast.py | 13 ++++++++++--- jedi/parser/tokenize.py | 1 + test/test_parser/test_fast_parser.py | 17 ++++++++--------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index a69a9d2f..44748e71 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -280,13 +280,22 @@ class FastParser(use_metaclass(CachedFastParser)): new_indent = False parentheses_level = 0 flow_indent = None + previous_line = None # All things within flows are simply being ignored. for i, l in enumerate(self._lines): + # Handle backslash newline escaping. + if l.endswith('\\\n') or l.endswith('\\\r\n'): + previous_line = l + continue + if previous_line is not None: + l = previous_line + l + previous_line = None + # check for dedents s = l.lstrip('\t \n\r') indent = len(l) - len(s) if not s or s[0] == '#': - current_lines.append(l) # just ignore comments and blank lines + current_lines.append(l) # Just ignore comments and blank lines continue if new_indent: @@ -379,8 +388,6 @@ class FastParser(use_metaclass(CachedFastParser)): % (self.module_path, self.number_parsers_used, self.number_of_splits)) - # print(self.parsers[0].module.get_code()) - def _get_node(self, source, parser_code, line_offset, nodes, no_docstr): """ Side effect: Alters the list of nodes. diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index d70c2784..2e7cdcc7 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -265,6 +265,7 @@ def generate_tokens(readline): break yield NAME, token, spos, prefix elif initial == '\\' and line[start:] == '\\\n': # continued stmt + additional_prefix += line[start:] continue else: if token in '([{': diff --git a/test/test_parser/test_fast_parser.py b/test/test_parser/test_fast_parser.py index 360d0eda..e6428656 100644 --- a/test/test_parser/test_fast_parser.py +++ b/test/test_parser/test_fast_parser.py @@ -329,12 +329,11 @@ def test_open_parentheses(): # get_code, because it isn't valid code. assert p.module.get_code() == '\n' + func assert p.number_of_splits == 2 - # CAVEAT: There are multiple splits, but because of the open parentheses, - # we just use one parser. We could theoretically break at the `def`, - # however that would require to also break properly at flow level. That is - # not possible because of ternary statements and would lead to strange - # behavior without much more knowledge. So we just keep it this way. assert p.number_parsers_used == 2 + cache.save_parser(None, None, p, pickling=False) + + # Now with a correct parser it should work perfectly well. + check_fp('isinstance()\n' + func, 1) def test_strange_parentheses(): @@ -349,7 +348,7 @@ def test_strange_parentheses(): def test_backslash(): - src = dedent("""\ + src = dedent(r""" a = 1\ if 1 else 2 def x(): @@ -357,8 +356,8 @@ def test_backslash(): """) check_fp(src, 2) - src = dedent("""\ - if 2: + src = dedent(r""" + def x(): a = 1\ if 1 else 2 def y(): @@ -366,7 +365,7 @@ def test_backslash(): """) # The dangling if leads to not splitting where we theoretically could # split. - check_fp(src, 1) + check_fp(src, 2) def test_incomplete_function():