diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index 672b7202..a313229f 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -240,9 +240,10 @@ class FastParser(use_metaclass(CachedFastParser)): def update(self, source): # For testing purposes: It is important that the number of parsers used - # can be minimized. With this variable we can test it. + # can be minimized. With these variables we can test against that. self.number_parsers_used = 0 self.number_of_splits = 0 + self.number_of_misses = 0 self.module.reset_caches() try: self._parse(source) @@ -285,7 +286,10 @@ class FastParser(use_metaclass(CachedFastParser)): for i, l in enumerate(self._lines): # Handle backslash newline escaping. if l.endswith('\\\n') or l.endswith('\\\r\n'): - previous_line = l + if previous_line is not None: + previous_line += l + else: + previous_line = l continue if previous_line is not None: l = previous_line + l @@ -371,6 +375,7 @@ class FastParser(use_metaclass(CachedFastParser)): # Means that some lines where not fully parsed. Parse it now. # This is a very rare case. Should only happens with very # strange code bits. + self.number_of_misses += 1 while last_end_line < next_line_offset + 1: line_offset = last_end_line - 1 # We could calculate the src in a more complicated way to @@ -383,7 +388,7 @@ class FastParser(use_metaclass(CachedFastParser)): last_end_line = self.current_node.parser.module.end_pos[0] debug.dbg('While parsing %s, line %s slowed down the fast parser.', - self.module_path, line_offset) + self.module_path, line_offset + 1) line_offset = next_line_offset start += len(code_part) @@ -501,7 +506,7 @@ class FastTokenizer(object): # Parentheses ignore the indentation rules. The other three stand for # new lines. if self.previous[0] in (NEWLINE, INDENT, DEDENT) \ - and not self._parentheses_level and typ != INDENT: + and not self._parentheses_level and typ not in (INDENT, DEDENT): # Check for NEWLINE, which symbolizes the indent. if not self._in_flow: if value in FLOWS: diff --git a/jedi/parser/pgen2/parse.py b/jedi/parser/pgen2/parse.py index c42a1485..5ae25d9a 100644 --- a/jedi/parser/pgen2/parse.py +++ b/jedi/parser/pgen2/parse.py @@ -121,7 +121,7 @@ class PgenParser(object): ilabel = self.classify(type, value, start_pos) except ParseError: # Currently we ignore tokens like `?`. - print('invalid token', tokenize.tok_name[type], value) + print('invalid token', tokenize.tok_name[type], repr(value)) return # Loop until the token is shifted; may raise exceptions diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 2e7cdcc7..53621034 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -265,8 +265,8 @@ def generate_tokens(readline): break yield NAME, token, spos, prefix elif initial == '\\' and line[start:] == '\\\n': # continued stmt - additional_prefix += line[start:] - continue + additional_prefix += prefix + line[start:] + break else: if token in '([{': paren_level += 1 diff --git a/test/test_parser/test_fast_parser.py b/test/test_parser/test_fast_parser.py index 7d0a5289..0324603b 100644 --- a/test/test_parser/test_fast_parser.py +++ b/test/test_parser/test_fast_parser.py @@ -76,7 +76,7 @@ def test_split_parts(): test('a\n', 'def b():\n pass\n', 'c\n') -def check_fp(src, number_parsers_used, number_of_splits=None): +def check_fp(src, number_parsers_used, number_of_splits=None, number_of_misses=0): if number_of_splits is None: number_of_splits = number_parsers_used @@ -88,6 +88,7 @@ def check_fp(src, number_parsers_used, number_of_splits=None): assert src == p.module.get_code() assert p.number_of_splits == number_of_splits assert p.number_parsers_used == number_parsers_used + assert p.number_of_misses == number_of_misses return p.module @@ -367,6 +368,18 @@ def test_backslash(): # split. check_fp(src, 2) + src = dedent(r""" + def first(): + if foo \ + and bar \ + or baz: + pass + def second(): + pass + """) + check_fp(src, 2) + + def test_fake_parentheses(): """ @@ -383,7 +396,7 @@ def test_fake_parentheses(): def z(): pass """) - check_fp(src, 3, 2) + check_fp(src, 3, 2, 1) def test_incomplete_function():