diff --git a/jedi/parser/fast.py b/jedi/parser/fast.py index 12765272..79093606 100644 --- a/jedi/parser/fast.py +++ b/jedi/parser/fast.py @@ -298,6 +298,7 @@ class FastParser(use_metaclass(CachedFastParser)): # For testing purposes: It is important that the number of parsers used # can be minimized. With this variable we can test it. self.number_parsers_used = 0 + self.number_of_splits = 0 self.module.reset_caches() try: self._parse(source) @@ -315,6 +316,7 @@ class FastParser(use_metaclass(CachedFastParser)): def gen_part(): text = '\n'.join(current_lines) del current_lines[:] + self.number_of_splits += 1 return text # Split only new lines. Distinction between \r\n is the tokenizer's @@ -594,17 +596,11 @@ class FastTokenizer(object): #self._parser_indent += 1 # new scope: must be higher #self._new_indent = True - if value != '@': - if self._first_stmt and not self._new_indent: - self._parser_indent = indent - self._first_stmt = False - - # Ignore closing parentheses, because they are all - # irrelevant for the indentation. - if value in '([{' and value: self._parentheses_level += 1 elif value in ')]}' and value: + # Ignore closing parentheses, because they are all + # irrelevant for the indentation. self._parentheses_level = max(self._parentheses_level - 1, 0) return current @@ -612,7 +608,10 @@ class FastTokenizer(object): if self._first_stmt: # Continue like nothing has happened, because we want to enter # the first class/function. - self._first_stmt = False + if self.current[1] != '@': + #if self._first_stmt and not self._new_indent: + #self._parser_indent = indent + self._first_stmt = False return self.current else: self._closed = True @@ -627,7 +626,10 @@ class FastTokenizer(object): elif not self._returned_endmarker: self._returned_endmarker = True # We're using the current prefix for the endmarker to not loose any - # information. - return ENDMARKER, '', start_pos, self.current[3] + # information. However we care about "lost" lines. The prefix of + # the current line (indent) will always be included in the current + # line. + prefix = re.sub('[^\n]+$', '', self.current[3]) + return ENDMARKER, '', start_pos, prefix else: raise StopIteration diff --git a/jedi/parser/tokenize.py b/jedi/parser/tokenize.py index 3240082c..1c53bc83 100644 --- a/jedi/parser/tokenize.py +++ b/jedi/parser/tokenize.py @@ -154,7 +154,11 @@ def generate_tokens(readline, line_offset=0): numchars = '0123456789' contstr = '' contline = None - new_line = False + # We start with a newline. This makes indent at the first position + # possible. It's not valid Python, but still better than an INDENT in the + # second line (and not in the first). This makes quite a few things in + # Jedi's fast parser possible. + new_line = True prefix = '' # Should never be required, but here for safety additional_prefix = '' while True: # loop over lines in stream diff --git a/test/test_parser/test_fast_parser.py b/test/test_parser/test_fast_parser.py index 21d5772d..46adb9e6 100644 --- a/test/test_parser/test_fast_parser.py +++ b/test/test_parser/test_fast_parser.py @@ -59,13 +59,17 @@ def test_carriage_return_splitting(): assert [n.value for lst in p.module.names_dict.values() for n in lst] == ['Foo'] -def check_fp(src, number_parsers_used): +def check_fp(src, number_parsers_used, number_of_splits=None): + if number_of_splits is None: + number_of_splits = number_parsers_used + p = FastParser(load_grammar(), u(src)) cache.save_parser(None, None, p, pickling=False) # TODO Don't change get_code, the whole thing should be the same. # -> Need to refactor the parser first, though. assert src == p.module.get_code() + assert p.number_of_splits == number_of_splits assert p.number_parsers_used == number_parsers_used return p.module @@ -77,18 +81,18 @@ def test_change_and_undo(): # Parse the function and a. check_fp(func_before + 'a', 2) # Parse just b. - check_fp(func_before + 'b', 1) + check_fp(func_before + 'b', 1, 2) # b has changed to a again, so parse that. - check_fp(func_before + 'a', 1) + check_fp(func_before + 'a', 1, 2) # Same as before no parsers should be used. - check_fp(func_before + 'a', 0) + check_fp(func_before + 'a', 0, 2) # Getting rid of an old parser: Still no parsers used. - check_fp('a', 0) + check_fp('a', 0, 1) # Now the file has completely change and we need to parse. - check_fp('b', 1) + check_fp('b', 1, 1) # And again. - check_fp('a', 1) + check_fp('a', 1, 1) def test_positions(): @@ -100,7 +104,7 @@ def test_positions(): assert m.start_pos == (1, 0) assert m.end_pos == (3, 1) - m = check_fp('a', 0) + m = check_fp('a', 0, 1) assert m.start_pos == (1, 0) assert m.end_pos == (1, 1) @@ -156,6 +160,26 @@ def test_func_with_if(): check_fp(src, 1) +def test_decorator(): + src = dedent("""\ + class Decorator(): + @memoize + def dec(self, a): + return a + """) + check_fp(src, 2) + + +def test_nested_funcs(): + src = dedent("""\ + def memoize(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + """) + check_fp(src, 3) + + def test_incomplete_function(): source = '''return ImportErr'''