Fix a failing test

Let consumers know that we have type annotations

.gitignore

@@ -11,3 +11,4 @@ parso.egg-info/
 /.cache/
 /.pytest_cache
 test/fuzz-redo.pickle
+/venv/

.travis.yml

@@ -6,10 +6,13 @@ python:
   - 3.5
   - 3.6
   - 3.7
-  - 3.8
+  - 3.8.2
+  - nightly
   - pypy2.7-6.0
   - pypy3.5-6.0
 matrix:
+  allow_failures:
+    - python: nightly
   include:
     - python: 3.5
       env: TOXENV=py35-coverage

AUTHORS.txt

@@ -50,6 +50,8 @@ Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
 Simon Ruggier (@sruggier)
 Élie Gouzien (@ElieGouzien)
 Tim Gates (@timgates42) <tim.gates@iress.com>
+Batuhan Taskaya (@isidentical) <isidentical@gmail.com>
+Jocelyn Boullier (@Kazy) <jocelyn@boullier.bzh>
 
 
 Note: (@user) means a github user name.

CHANGELOG.rst

@@ -3,6 +3,29 @@
 Changelog
 ---------
 
+0.7.1 (2020-07-24)
+++++++++++++++++++
+
+- Fixed a couple of smaller bugs (mostly syntax error detection in
+  ``Grammar.iter_errors``)
+
+This is going to be the last release that supports Python 2.7, 3.4 and 3.5.
+
+0.7.0 (2020-04-13)
+++++++++++++++++++
+
+- Fix a lot of annoying bugs in the diff parser. The fuzzer did not find
+  issues anymore even after running it for more than 24 hours (500k tests).
+- Small grammar change: suites can now contain newlines even after a newline.
+  This should really not matter if you don't use error recovery. It allows for
+  nicer error recovery.
+
+0.6.2 (2020-02-27)
+++++++++++++++++++
+
+- Bugfixes
+- Add Grammar.refactor (might still be subject to change until 0.7.0)
+
 0.6.1 (2020-02-03)
 ++++++++++++++++++
 

README.rst

@@ -11,6 +11,10 @@ parso - A Python Parser
     :target: https://coveralls.io/github/davidhalter/parso?branch=master
     :alt: Coverage Status
 
+.. image:: https://pepy.tech/badge/parso
+    :target: https://pepy.tech/project/parso
+    :alt: PyPI Downloads
+
 .. image:: https://raw.githubusercontent.com/davidhalter/parso/master/docs/_static/logo_characters.png
 
 Parso is a Python parser that supports error recovery and round-trip parsing

conftest.py

@@ -87,12 +87,12 @@ def pytest_configure(config):
         root = logging.getLogger()
         root.setLevel(logging.DEBUG)
 
-        ch = logging.StreamHandler(sys.stdout)
-        ch.setLevel(logging.DEBUG)
+        #ch = logging.StreamHandler(sys.stdout)
+        #ch.setLevel(logging.DEBUG)
         #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
         #ch.setFormatter(formatter)
 
-        root.addHandler(ch)
+        #root.addHandler(ch)
 
 
 class Checker():
@@ -158,8 +158,17 @@ def works_ge_py35(each_version):
     version_info = parse_version_string(each_version)
     return Checker(each_version, version_info >= (3, 5))
 
+@pytest.fixture
+def works_ge_py36(each_version):
+    version_info = parse_version_string(each_version)
+    return Checker(each_version, version_info >= (3, 6))
 
 @pytest.fixture
 def works_ge_py38(each_version):
     version_info = parse_version_string(each_version)
     return Checker(each_version, version_info >= (3, 8))
+
+@pytest.fixture
+def works_ge_py39(each_version):
+    version_info = parse_version_string(each_version)
+    return Checker(each_version, version_info >= (3, 9))

deploy-master.sh

@@ -26,7 +26,7 @@ git checkout $BRANCH
 tox
 
 # Create tag
-tag=v$(python -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
+tag=v$(python3 -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
 
 master_ref=$(git show-ref -s heads/$BRANCH)
 tag_ref=$(git show-ref -s $tag || true)
@@ -43,7 +43,7 @@ fi
 # Package and upload to PyPI
 #rm -rf dist/ - Not needed anymore, because the folder is never reused.
 echo `pwd`
-python setup.py sdist bdist_wheel
+python3 setup.py sdist bdist_wheel
 # Maybe do a pip install twine before.
 twine upload dist/*
 

parso/__init__.py

@@ -43,7 +43,7 @@ from parso.grammar import Grammar, load_grammar
 from parso.utils import split_lines, python_bytes_to_unicode
 
 
-__version__ = '0.6.1'
+__version__ = '0.7.1'
 
 
 def parse(code=None, **kwargs):

parso/_compatibility.py

@@ -2,6 +2,7 @@
 To ensure compatibility from Python ``2.7`` - ``3.3``, a module has been
 created. Clearly there is huge need to use conforming syntax.
 """
+import os
 import sys
 import platform
 
@@ -44,11 +45,17 @@ def u(string):
 
 
 try:
-    # Python 2.7
+    # Python 3.3+
     FileNotFoundError = FileNotFoundError
 except NameError:
+    # Python 2.7 (both IOError + OSError)
+    FileNotFoundError = EnvironmentError
+try:
     # Python 3.3+
-    FileNotFoundError = IOError
+    PermissionError = PermissionError
+except NameError:
+    # Python 2.7 (both IOError + OSError)
+    PermissionError = EnvironmentError
 
 
 def utf8_repr(func):
@@ -67,3 +74,28 @@ def utf8_repr(func):
         return func
     else:
         return wrapper
+
+
+if sys.version_info < (3, 5):
+    """
+    A super-minimal shim around listdir that behave like
+    scandir for the information we need.
+    """
+    class _DirEntry:
+
+        def __init__(self, name, basepath):
+            self.name = name
+            self.basepath = basepath
+
+        @property
+        def path(self):
+            return os.path.join(self.basepath, self.name)
+
+        def stat(self):
+            # won't follow symlinks
+            return os.lstat(os.path.join(self.basepath, self.name))
+
+    def scandir(dir):
+        return [_DirEntry(name, dir) for name in os.listdir(dir)]
+else:
+    from os import scandir

parso/cache.py

@@ -7,13 +7,15 @@ import shutil
 import platform
 import errno
 import logging
+import warnings
 
 try:
     import cPickle as pickle
 except:
     import pickle
 
-from parso._compatibility import FileNotFoundError
+from parso._compatibility import FileNotFoundError, PermissionError, scandir
+from parso.file_io import FileIO
 
 LOG = logging.getLogger(__name__)
 
@@ -21,6 +23,13 @@ _CACHED_FILE_MINIMUM_SURVIVAL = 60 * 10  # 10 minutes
 """
 Cached files should survive at least a few minutes.
 """
+
+_CACHED_FILE_MAXIMUM_SURVIVAL = 60 * 60 * 24 * 30
+"""
+Maximum time for a cached file to survive if it is not
+accessed within.
+"""
+
 _CACHED_SIZE_TRIGGER = 600
 """
 This setting limits the amount of cached files. It's basically a way to start
@@ -63,7 +72,8 @@ http://docs.python.org/3/library/sys.html#sys.implementation
 
 def _get_default_cache_path():
     if platform.system().lower() == 'windows':
-        dir_ = os.path.join(os.getenv('LOCALAPPDATA') or '~', 'Parso', 'Parso')
+        dir_ = os.path.join(os.getenv('LOCALAPPDATA')
+                            or os.path.expanduser('~'), 'Parso', 'Parso')
     elif platform.system().lower() == 'darwin':
         dir_ = os.path.join('~', 'Library', 'Caches', 'Parso')
     else:
@@ -81,6 +91,19 @@ On Linux, if environment variable ``$XDG_CACHE_HOME`` is set,
 ``$XDG_CACHE_HOME/parso`` is used instead of the default one.
 """
 
+_CACHE_CLEAR_THRESHOLD = 60 * 60 * 24
+
+def _get_cache_clear_lock(cache_path = None):
+    """
+    The path where the cache lock is stored.
+
+    Cache lock will prevent continous cache clearing and only allow garbage
+    collection once a day (can be configured in _CACHE_CLEAR_THRESHOLD).
+    """
+    cache_path = cache_path or _get_default_cache_path()
+    return FileIO(os.path.join(cache_path, "PARSO-CACHE-LOCK"))
+
+
 parser_cache = {}
 
 
@@ -160,7 +183,7 @@ def _set_cache_item(hashed_grammar, path, module_cache_item):
     parser_cache.setdefault(hashed_grammar, {})[path] = module_cache_item
 
 
-def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None):
+def try_to_save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None):
     path = file_io.path
     try:
         p_time = None if path is None else file_io.get_last_modified()
@@ -171,7 +194,18 @@ def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_pat
     item = _NodeCacheItem(module, lines, p_time)
     _set_cache_item(hashed_grammar, path, item)
     if pickling and path is not None:
-        _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path)
+        try:
+            _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path)
+        except PermissionError:
+            # It's not really a big issue if the cache cannot be saved to the
+            # file system. It's still in RAM in that case. However we should
+            # still warn the user that this is happening.
+            warnings.warn(
+                'Tried to save a file to %s, but got permission denied.',
+                Warning
+            )
+        else:
+            _remove_cache_and_update_lock(cache_path=cache_path)
 
 
 def _save_to_file_system(hashed_grammar, path, item, cache_path=None):
@@ -186,6 +220,46 @@ def clear_cache(cache_path=None):
     parser_cache.clear()
 
 
+def clear_inactive_cache(
+    cache_path=None,
+    inactivity_threshold=_CACHED_FILE_MAXIMUM_SURVIVAL,
+):
+    if cache_path is None:
+        cache_path = _get_default_cache_path()
+    if not os.path.exists(cache_path):
+        return False
+    for version_path in os.listdir(cache_path):
+        version_path = os.path.join(cache_path, version_path)
+        if not os.path.isdir(version_path):
+            continue
+        for file in scandir(version_path):
+            if (
+                file.stat().st_atime + _CACHED_FILE_MAXIMUM_SURVIVAL
+                <= time.time()
+            ):
+                try:
+                    os.remove(file.path)
+                except OSError: # silently ignore all failures
+                    continue
+    else:
+        return True
+
+
+def _remove_cache_and_update_lock(cache_path = None):
+    lock = _get_cache_clear_lock(cache_path=cache_path)
+    clear_lock_time = lock.get_last_modified()
+    if (
+        clear_lock_time is None # first time
+        or clear_lock_time + _CACHE_CLEAR_THRESHOLD <= time.time()
+    ):
+        if not lock._touch():
+            # First make sure that as few as possible other cleanup jobs also
+            # get started. There is still a race condition but it's probably
+            # not a big problem.
+            return False
+
+        clear_inactive_cache(cache_path = cache_path)
+
 def _get_hashed_path(hashed_grammar, path, cache_path=None):
     directory = _get_cache_directory_path(cache_path=cache_path)
 

parso/file_io.py

@@ -1,4 +1,5 @@
 import os
+from parso._compatibility import FileNotFoundError
 
 
 class FileIO(object):
@@ -22,6 +23,17 @@ class FileIO(object):
             # Might raise FileNotFoundError, OSError for Python 2
             return None
 
+    def _touch(self):
+        try:
+            os.utime(self.path, None)
+        except FileNotFoundError:
+            try:
+                file = open(self.path, 'a')
+                file.close()
+            except (OSError, IOError):  # TODO Maybe log this?
+                return False
+        return True
+
     def __repr__(self):
         return '%s(%s)' % (self.__class__.__name__, self.path)
 

parso/grammar.py

@@ -7,12 +7,13 @@ from parso.utils import split_lines, python_bytes_to_unicode, parse_version_stri
 from parso.python.diff import DiffParser
 from parso.python.tokenize import tokenize_lines, tokenize
 from parso.python.token import PythonTokenTypes
-from parso.cache import parser_cache, load_module, save_module
+from parso.cache import parser_cache, load_module, try_to_save_module
 from parso.parser import BaseParser
 from parso.python.parser import Parser as PythonParser
 from parso.python.errors import ErrorFinderConfig
 from parso.python import pep8
 from parso.file_io import FileIO, KnownContentFileIO
+from parso.normalizer import RefactoringNormalizer
 
 _loaded_grammars = {}
 
@@ -131,13 +132,13 @@ class Grammar(object):
                     old_lines=old_lines,
                     new_lines=lines
                 )
-                save_module(self._hashed, file_io, new_node, lines,
+                try_to_save_module(self._hashed, file_io, new_node, lines,
                             # Never pickle in pypy, it's slow as hell.
                             pickling=cache and not is_pypy,
                             cache_path=cache_path)
                 return new_node
 
-        tokens = self._tokenizer(lines, start_pos)
+        tokens = self._tokenizer(lines, start_pos=start_pos)
 
         p = self._parser(
             self._pgen_grammar,
@@ -147,7 +148,7 @@ class Grammar(object):
         root_node = p.parse(tokens=tokens)
 
         if cache or diff_cache:
-            save_module(self._hashed, file_io, root_node, lines,
+            try_to_save_module(self._hashed, file_io, root_node, lines,
                         # Never pickle in pypy, it's slow as hell.
                         pickling=cache and not is_pypy,
                         cache_path=cache_path)
@@ -170,6 +171,9 @@ class Grammar(object):
 
         return self._get_normalizer_issues(node, self._error_normalizer_config)
 
+    def refactor(self, base_node, node_to_str_map):
+        return RefactoringNormalizer(node_to_str_map).walk(base_node)
+
     def _get_normalizer(self, normalizer_config):
         if normalizer_config is None:
             normalizer_config = self._default_normalizer_config
@@ -211,8 +215,8 @@ class PythonGrammar(Grammar):
         )
         self.version_info = version_info
 
-    def _tokenize_lines(self, lines, start_pos):
-        return tokenize_lines(lines, self.version_info, start_pos=start_pos)
+    def _tokenize_lines(self, lines, **kwargs):
+        return tokenize_lines(lines, self.version_info, **kwargs)
 
     def _tokenize(self, code):
         # Used by Jedi.
@@ -248,7 +252,7 @@ def load_grammar(**kwargs):
                     grammar = PythonGrammar(version_info, bnf_text)
                     return _loaded_grammars.setdefault(path, grammar)
                 except FileNotFoundError:
-                    message = "Python version %s is currently not supported." % version
+                    message = "Python version %s.%s is currently not supported." % (version_info.major, version_info.minor)
                     raise NotImplementedError(message)
         else:
             raise NotImplementedError("No support for language %s." % language)

parso/normalizer.py

@@ -12,6 +12,9 @@ class _NormalizerMeta(type):
 
 
 class Normalizer(use_metaclass(_NormalizerMeta)):
+    _rule_type_instances = {}
+    _rule_value_instances = {}
+
     def __init__(self, grammar, config):
         self.grammar = grammar
         self._config = config
@@ -160,7 +163,7 @@ class Rule(object):
     def get_node(self, node):
         return node
 
-    def _get_message(self, message):
+    def _get_message(self, message, node):
         if message is None:
             message = self.message
             if message is None:
@@ -173,7 +176,7 @@ class Rule(object):
             if code is None:
                 raise ValueError("The error code on the class is not set.")
 
-        message = self._get_message(message)
+        message = self._get_message(message, node)
 
         self._normalizer.add_issue(node, code, message)
 
@@ -181,3 +184,20 @@ class Rule(object):
         if self.is_issue(node):
             issue_node = self.get_node(node)
             self.add_issue(issue_node)
+
+
+class RefactoringNormalizer(Normalizer):
+    def __init__(self, node_to_str_map):
+        self._node_to_str_map = node_to_str_map
+
+    def visit(self, node):
+        try:
+            return self._node_to_str_map[node]
+        except KeyError:
+            return super(RefactoringNormalizer, self).visit(node)
+
+    def visit_leaf(self, leaf):
+        try:
+            return self._node_to_str_map[leaf]
+        except KeyError:
+            return super(RefactoringNormalizer, self).visit_leaf(leaf)

parso/parser.py

@@ -134,7 +134,7 @@ class BaseParser(object):
                 # However, the error recovery might have added the token again, if
                 # the stack is empty, we're fine.
                 raise InternalParseError(
-                    "incomplete input", token.type, token.value, token.start_pos
+                    "incomplete input", token.type, token.string, token.start_pos
                 )
 
             if len(self.stack) > 1:

parso/pgen2/generator.py

@@ -212,7 +212,8 @@ def _dump_nfa(start, finish):
     todo = [start]
     for i, state in enumerate(todo):
         print("  State", i, state is finish and "(final)" or "")
-        for label, next_ in state.arcs:
+        for arc in state.arcs:
+            label, next_ = arc.nonterminal_or_string, arc.next
             if next_ in todo:
                 j = todo.index(next_)
             else:
@@ -244,7 +245,7 @@ def generate_grammar(bnf_grammar, token_namespace):
     rule_to_dfas = {}
     start_nonterminal = None
     for nfa_a, nfa_z in GrammarParser(bnf_grammar).parse():
-        #_dump_nfa(a, z)
+        #_dump_nfa(nfa_a, nfa_z)
         dfas = _make_dfas(nfa_a, nfa_z)
         #_dump_dfas(dfas)
         # oldlen = len(dfas)

parso/python/diff.py

@@ -1,9 +1,29 @@
 """
-Basically a contains parser that is faster, because it tries to parse only
-parts and if anything changes, it only reparses the changed parts.
+The diff parser is trying to be a faster version of the normal parser by trying
+to reuse the nodes of a previous pass over the same file. This is also called
+incremental parsing in parser literature. The difference is mostly that with
+incremental parsing you get a range that needs to be reparsed. Here we
+calculate that range ourselves by using difflib. After that it's essentially
+incremental parsing.
 
-It works with a simple diff in the beginning and will try to reuse old parser
-fragments.
+The biggest issue of this approach is that we reuse nodes in a mutable way. The
+intial design and idea is quite problematic for this parser, but it is also
+pretty fast. Measurements showed that just copying nodes in Python is simply
+quite a bit slower (especially for big files >3 kLOC). Therefore we did not
+want to get rid of the mutable nodes, since this is usually not an issue.
+
+This is by far the hardest software I ever wrote, exactly because the initial
+design is crappy. When you have to account for a lot of mutable state, it
+creates a ton of issues that you would otherwise not have. This file took
+probably 3-6 months to write, which is insane for a parser.
+
+There is a fuzzer in that helps test this whole thing. Please use it if you
+make changes here. If you run the fuzzer like::
+
+    test/fuzz_diff_parser.py random -n 100000
+
+you can be pretty sure that everything is still fine. I sometimes run the
+fuzzer up to 24h to make sure everything is still ok.
 """
 import re
 import difflib
@@ -13,7 +33,7 @@ import logging
 from parso.utils import split_lines
 from parso.python.parser import Parser
 from parso.python.tree import EndMarker
-from parso.python.tokenize import PythonToken
+from parso.python.tokenize import PythonToken, BOM_UTF8_STRING
 from parso.python.token import PythonTokenTypes
 
 LOG = logging.getLogger(__name__)
@@ -21,21 +41,37 @@ DEBUG_DIFF_PARSER = False
 
 _INDENTATION_TOKENS = 'INDENT', 'ERROR_DEDENT', 'DEDENT'
 
+NEWLINE = PythonTokenTypes.NEWLINE
+DEDENT = PythonTokenTypes.DEDENT
+NAME = PythonTokenTypes.NAME
+ERROR_DEDENT = PythonTokenTypes.ERROR_DEDENT
+ENDMARKER = PythonTokenTypes.ENDMARKER
+
+
+def _is_indentation_error_leaf(node):
+    return node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS
+
 
 def _get_previous_leaf_if_indentation(leaf):
-    while leaf and leaf.type == 'error_leaf' \
-            and leaf.token_type in _INDENTATION_TOKENS:
+    while leaf and _is_indentation_error_leaf(leaf):
         leaf = leaf.get_previous_leaf()
     return leaf
 
 
 def _get_next_leaf_if_indentation(leaf):
-    while leaf and leaf.type == 'error_leaf' \
-            and leaf.token_type in _INDENTATION_TOKENS:
-        leaf = leaf.get_previous_leaf()
+    while leaf and _is_indentation_error_leaf(leaf):
+        leaf = leaf.get_next_leaf()
     return leaf
 
 
+def _get_suite_indentation(tree_node):
+    return _get_indentation(tree_node.children[1])
+
+
+def _get_indentation(tree_node):
+    return tree_node.start_pos[1]
+
+
 def _assert_valid_graph(node):
     """
     Checks if the parent/children relationship is correct.
@@ -70,6 +106,10 @@ def _assert_valid_graph(node):
             actual = line, len(splitted[-1])
         else:
             actual = previous_start_pos[0], previous_start_pos[1] + len(content)
+            if content.startswith(BOM_UTF8_STRING) \
+                    and node.get_start_pos_of_prefix() == (1, 0):
+                # Remove the byte order mark
+                actual = actual[0], actual[1] - 1
 
         assert node.start_pos == actual, (node.start_pos, actual)
     else:
@@ -78,6 +118,26 @@ def _assert_valid_graph(node):
             _assert_valid_graph(child)
 
 
+def _assert_nodes_are_equal(node1, node2):
+    try:
+        children1 = node1.children
+    except AttributeError:
+        assert not hasattr(node2, 'children'), (node1, node2)
+        assert node1.value == node2.value, (node1, node2)
+        assert node1.type == node2.type, (node1, node2)
+        assert node1.prefix == node2.prefix, (node1, node2)
+        assert node1.start_pos == node2.start_pos, (node1, node2)
+        return
+    else:
+        try:
+            children2 = node2.children
+        except AttributeError:
+            assert False, (node1, node2)
+    for n1, n2 in zip(children1, children2):
+        _assert_nodes_are_equal(n1, n2)
+    assert len(children1) == len(children2), '\n' + repr(children1) + '\n' + repr(children2)
+
+
 def _get_debug_error_message(module, old_lines, new_lines):
     current_lines = split_lines(module.get_code(), keepends=True)
     current_diff = difflib.unified_diff(new_lines, current_lines)
@@ -95,6 +155,15 @@ def _get_last_line(node_or_leaf):
     if _ends_with_newline(last_leaf):
         return last_leaf.start_pos[0]
     else:
+        n = last_leaf.get_next_leaf()
+        if n.type == 'endmarker' and '\n' in n.prefix:
+            # This is a very special case and has to do with error recovery in
+            # Parso. The problem is basically that there's no newline leaf at
+            # the end sometimes (it's required in the grammar, but not needed
+            # actually before endmarker, CPython just adds a newline to make
+            # source code pass the parser, to account for that Parso error
+            # recovery allows small_stmt instead of simple_stmt).
+            return last_leaf.end_pos[0] + 1
         return last_leaf.end_pos[0]
 
 
@@ -233,7 +302,7 @@ class DiffParser(object):
 
             if operation == 'equal':
                 line_offset = j1 - i1
-                self._copy_from_old_parser(line_offset, i2, j2)
+                self._copy_from_old_parser(line_offset, i1 + 1, i2, j2)
             elif operation == 'replace':
                 self._parse(until_line=j2)
             elif operation == 'insert':
@@ -249,8 +318,14 @@ class DiffParser(object):
             # If there is reasonable suspicion that the diff parser is not
             # behaving well, this should be enabled.
             try:
-                assert self._module.get_code() == ''.join(new_lines)
+                code = ''.join(new_lines)
+                assert self._module.get_code() == code
                 _assert_valid_graph(self._module)
+                without_diff_parser_module = Parser(
+                    self._pgen_grammar,
+                    error_recovery=True
+                ).parse(self._tokenizer(new_lines))
+                _assert_nodes_are_equal(self._module, without_diff_parser_module)
             except AssertionError:
                 print(_get_debug_error_message(self._module, old_lines, new_lines))
                 raise
@@ -268,7 +343,7 @@ class DiffParser(object):
         if self._module.get_code() != ''.join(lines_new):
             LOG.warning('parser issue:\n%s\n%s', ''.join(old_lines), ''.join(lines_new))
 
-    def _copy_from_old_parser(self, line_offset, until_line_old, until_line_new):
+    def _copy_from_old_parser(self, line_offset, start_line_old, until_line_old, until_line_new):
         last_until_line = -1
         while until_line_new > self._nodes_tree.parsed_until_line:
             parsed_until_line_old = self._nodes_tree.parsed_until_line - line_offset
@@ -282,12 +357,18 @@ class DiffParser(object):
                 p_children = line_stmt.parent.children
                 index = p_children.index(line_stmt)
 
-                from_ = self._nodes_tree.parsed_until_line + 1
-                copied_nodes = self._nodes_tree.copy_nodes(
-                    p_children[index:],
-                    until_line_old,
-                    line_offset
-                )
+                if start_line_old == 1 \
+                        and p_children[0].get_first_leaf().prefix.startswith(BOM_UTF8_STRING):
+                    # If there's a BOM in the beginning, just reparse. It's too
+                    # complicated to account for it otherwise.
+                    copied_nodes = []
+                else:
+                    from_ = self._nodes_tree.parsed_until_line + 1
+                    copied_nodes = self._nodes_tree.copy_nodes(
+                        p_children[index:],
+                        until_line_old,
+                        line_offset
+                    )
                 # Match all the nodes that are in the wanted range.
                 if copied_nodes:
                     self._copy_count += 1
@@ -333,7 +414,10 @@ class DiffParser(object):
             node = self._try_parse_part(until_line)
             nodes = node.children
 
-            self._nodes_tree.add_parsed_nodes(nodes)
+            self._nodes_tree.add_parsed_nodes(nodes, self._keyword_token_indents)
+            if self._replace_tos_indent is not None:
+                self._nodes_tree.indents[-1] = self._replace_tos_indent
+
             LOG.debug(
                 'parse_part from %s to %s (to %s in part parser)',
                 nodes[0].get_start_pos_of_prefix()[0],
@@ -369,34 +453,39 @@ class DiffParser(object):
         return self._active_parser.parse(tokens=tokens)
 
     def _diff_tokenize(self, lines, until_line, line_offset=0):
-        is_first_token = True
-        omitted_first_indent = False
-        indents = []
-        tokens = self._tokenizer(lines, (1, 0))
-        stack = self._active_parser.stack
-        for typ, string, start_pos, prefix in tokens:
-            start_pos = start_pos[0] + line_offset, start_pos[1]
-            if typ == PythonTokenTypes.INDENT:
-                indents.append(start_pos[1])
-                if is_first_token:
-                    omitted_first_indent = True
-                    # We want to get rid of indents that are only here because
-                    # we only parse part of the file. These indents would only
-                    # get parsed as error leafs, which doesn't make any sense.
-                    is_first_token = False
-                    continue
-            is_first_token = False
+        was_newline = False
+        indents = self._nodes_tree.indents
+        initial_indentation_count = len(indents)
 
-            # In case of omitted_first_indent, it might not be dedented fully.
-            # However this is a sign for us that a dedent happened.
-            if typ == PythonTokenTypes.DEDENT \
-                    or typ == PythonTokenTypes.ERROR_DEDENT \
-                    and omitted_first_indent and len(indents) == 1:
-                indents.pop()
-                if omitted_first_indent and not indents:
+        tokens = self._tokenizer(
+            lines,
+            start_pos=(line_offset + 1, 0),
+            indents=indents,
+            is_first_token=line_offset == 0,
+        )
+        stack = self._active_parser.stack
+        self._replace_tos_indent = None
+        self._keyword_token_indents = {}
+        # print('start', line_offset + 1, indents)
+        for token in tokens:
+            # print(token, indents)
+            typ = token.type
+            if typ == DEDENT:
+                if len(indents) < initial_indentation_count:
                     # We are done here, only thing that can come now is an
                     # endmarker or another dedented code block.
-                    typ, string, start_pos, prefix = next(tokens)
+                    while True:
+                        typ, string, start_pos, prefix = token = next(tokens)
+                        if typ in (DEDENT, ERROR_DEDENT):
+                            if typ == ERROR_DEDENT:
+                                # We want to force an error dedent in the next
+                                # parser/pass. To make this possible we just
+                                # increase the location by one.
+                                self._replace_tos_indent = start_pos[1] + 1
+                                pass
+                        else:
+                            break
+
                     if '\n' in prefix or '\r' in prefix:
                         prefix = re.sub(r'[^\n\r]+\Z', '', prefix)
                     else:
@@ -404,36 +493,38 @@ class DiffParser(object):
                         if start_pos[1] - len(prefix) == 0:
                             prefix = ''
                     yield PythonToken(
-                        PythonTokenTypes.ENDMARKER, '',
-                        (start_pos[0] + line_offset, 0),
+                        ENDMARKER, '',
+                        start_pos,
                         prefix
                     )
                     break
-            elif typ == PythonTokenTypes.NEWLINE and start_pos[0] >= until_line:
-                yield PythonToken(typ, string, start_pos, prefix)
-                # Check if the parser is actually in a valid suite state.
-                if _suite_or_file_input_is_valid(self._pgen_grammar, stack):
-                    start_pos = start_pos[0] + 1, 0
-                    while len(indents) > int(omitted_first_indent):
-                        indents.pop()
-                        yield PythonToken(PythonTokenTypes.DEDENT, '', start_pos, '')
+            elif typ == NEWLINE and token.start_pos[0] >= until_line:
+                was_newline = True
+            elif was_newline:
+                was_newline = False
+                if len(indents) == initial_indentation_count:
+                    # Check if the parser is actually in a valid suite state.
+                    if _suite_or_file_input_is_valid(self._pgen_grammar, stack):
+                        yield PythonToken(ENDMARKER, '', token.start_pos, '')
+                        break
 
-                    yield PythonToken(PythonTokenTypes.ENDMARKER, '', start_pos, '')
-                    break
-                else:
-                    continue
+            if typ == NAME and token.string in ('class', 'def'):
+                self._keyword_token_indents[token.start_pos] = list(indents)
 
-            yield PythonToken(typ, string, start_pos, prefix)
+            yield token
 
 
 class _NodesTreeNode(object):
-    _ChildrenGroup = namedtuple('_ChildrenGroup', 'prefix children line_offset last_line_offset_leaf')
+    _ChildrenGroup = namedtuple(
+        '_ChildrenGroup',
+        'prefix children line_offset last_line_offset_leaf')
 
-    def __init__(self, tree_node, parent=None):
+    def __init__(self, tree_node, parent=None, indentation=0):
         self.tree_node = tree_node
         self._children_groups = []
         self.parent = parent
         self._node_children = []
+        self.indentation = indentation
 
     def finish(self):
         children = []
@@ -461,10 +552,13 @@ class _NodesTreeNode(object):
     def add_child_node(self, child_node):
         self._node_children.append(child_node)
 
-    def add_tree_nodes(self, prefix, children, line_offset=0, last_line_offset_leaf=None):
+    def add_tree_nodes(self, prefix, children, line_offset=0,
+                       last_line_offset_leaf=None):
         if last_line_offset_leaf is None:
             last_line_offset_leaf = children[-1].get_last_leaf()
-        group = self._ChildrenGroup(prefix, children, line_offset, last_line_offset_leaf)
+        group = self._ChildrenGroup(
+            prefix, children, line_offset, last_line_offset_leaf
+        )
         self._children_groups.append(group)
 
     def get_last_line(self, suffix):
@@ -491,6 +585,9 @@ class _NodesTreeNode(object):
             return max(line, self._node_children[-1].get_last_line(suffix))
         return line
 
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
+
 
 class _NodesTree(object):
     def __init__(self, module):
@@ -499,34 +596,19 @@ class _NodesTree(object):
         self._module = module
         self._prefix_remainder = ''
         self.prefix = ''
+        self.indents = [0]
 
     @property
     def parsed_until_line(self):
         return self._working_stack[-1].get_last_line(self.prefix)
 
-    def _get_insertion_node(self, indentation_node):
-        indentation = indentation_node.start_pos[1]
-
-        # find insertion node
-        while True:
-            node = self._working_stack[-1]
-            tree_node = node.tree_node
-            if tree_node.type == 'suite':
-                # A suite starts with NEWLINE, ...
-                node_indentation = tree_node.children[1].start_pos[1]
-
-                if indentation >= node_indentation:  # Not a Dedent
-                    # We might be at the most outer layer: modules. We
-                    # don't want to depend on the first statement
-                    # having the right indentation.
-                    return node
-
-            elif tree_node.type == 'file_input':
+    def _update_insertion_node(self, indentation):
+        for node in reversed(list(self._working_stack)):
+            if node.indentation < indentation or node is self._working_stack[0]:
                 return node
-
             self._working_stack.pop()
 
-    def add_parsed_nodes(self, tree_nodes):
+    def add_parsed_nodes(self, tree_nodes, keyword_token_indents):
         old_prefix = self.prefix
         tree_nodes = self._remove_endmarker(tree_nodes)
         if not tree_nodes:
@@ -535,23 +617,27 @@ class _NodesTree(object):
 
         assert tree_nodes[0].type != 'newline'
 
-        node = self._get_insertion_node(tree_nodes[0])
+        node = self._update_insertion_node(tree_nodes[0].start_pos[1])
         assert node.tree_node.type in ('suite', 'file_input')
         node.add_tree_nodes(old_prefix, tree_nodes)
         # tos = Top of stack
-        self._update_tos(tree_nodes[-1])
+        self._update_parsed_node_tos(tree_nodes[-1], keyword_token_indents)
 
-    def _update_tos(self, tree_node):
-        if tree_node.type in ('suite', 'file_input'):
-            new_tos = _NodesTreeNode(tree_node)
+    def _update_parsed_node_tos(self, tree_node, keyword_token_indents):
+        if tree_node.type == 'suite':
+            def_leaf = tree_node.parent.children[0]
+            new_tos = _NodesTreeNode(
+                tree_node,
+                indentation=keyword_token_indents[def_leaf.start_pos][-1],
+            )
             new_tos.add_tree_nodes('', list(tree_node.children))
 
             self._working_stack[-1].add_child_node(new_tos)
             self._working_stack.append(new_tos)
 
-            self._update_tos(tree_node.children[-1])
+            self._update_parsed_node_tos(tree_node.children[-1], keyword_token_indents)
         elif _func_or_class_has_suite(tree_node):
-            self._update_tos(tree_node.children[-1])
+            self._update_parsed_node_tos(tree_node.children[-1], keyword_token_indents)
 
     def _remove_endmarker(self, tree_nodes):
         """
@@ -561,7 +647,8 @@ class _NodesTree(object):
         is_endmarker = last_leaf.type == 'endmarker'
         self._prefix_remainder = ''
         if is_endmarker:
-            separation = max(last_leaf.prefix.rfind('\n'), last_leaf.prefix.rfind('\r'))
+            prefix = last_leaf.prefix
+            separation = max(prefix.rfind('\n'), prefix.rfind('\r'))
             if separation > -1:
                 # Remove the whitespace part of the prefix after a newline.
                 # That is not relevant if parentheses were opened. Always parse
@@ -577,6 +664,26 @@ class _NodesTree(object):
             tree_nodes = tree_nodes[:-1]
         return tree_nodes
 
+    def _get_matching_indent_nodes(self, tree_nodes, is_new_suite):
+        # There might be a random dedent where we have to stop copying.
+        # Invalid indents are ok, because the parser handled that
+        # properly before. An invalid dedent can happen, because a few
+        # lines above there was an invalid indent.
+        node_iterator = iter(tree_nodes)
+        if is_new_suite:
+            yield next(node_iterator)
+
+        first_node = next(node_iterator)
+        indent = _get_indentation(first_node)
+        if not is_new_suite and indent not in self.indents:
+            return
+        yield first_node
+
+        for n in node_iterator:
+            if _get_indentation(n) != indent:
+                return
+            yield n
+
     def copy_nodes(self, tree_nodes, until_line, line_offset):
         """
         Copies tree nodes from the old parser tree.
@@ -588,19 +695,38 @@ class _NodesTree(object):
             # issues.
             return []
 
-        self._get_insertion_node(tree_nodes[0])
+        indentation = _get_indentation(tree_nodes[0])
+        old_working_stack = list(self._working_stack)
+        old_prefix = self.prefix
+        old_indents = self.indents
+        self.indents = [i for i in self.indents if i <= indentation]
 
-        new_nodes, self._working_stack, self.prefix = self._copy_nodes(
+        self._update_insertion_node(indentation)
+
+        new_nodes, self._working_stack, self.prefix, added_indents = self._copy_nodes(
             list(self._working_stack),
             tree_nodes,
             until_line,
             line_offset,
             self.prefix,
         )
+        if new_nodes:
+            self.indents += added_indents
+        else:
+            self._working_stack = old_working_stack
+            self.prefix = old_prefix
+            self.indents = old_indents
         return new_nodes
 
-    def _copy_nodes(self, working_stack, nodes, until_line, line_offset, prefix=''):
+    def _copy_nodes(self, working_stack, nodes, until_line, line_offset,
+                    prefix='', is_nested=False):
         new_nodes = []
+        added_indents = []
+
+        nodes = list(self._get_matching_indent_nodes(
+            nodes,
+            is_new_suite=is_nested,
+        ))
 
         new_prefix = ''
         for node in nodes:
@@ -620,26 +746,83 @@ class _NodesTree(object):
                 if _func_or_class_has_suite(node):
                     new_nodes.append(node)
                 break
+            try:
+                c = node.children
+            except AttributeError:
+                pass
+            else:
+                # This case basically appears with error recovery of one line
+                # suites like `def foo(): bar.-`. In this case we might not
+                # include a newline in the statement and we need to take care
+                # of that.
+                n = node
+                if n.type == 'decorated':
+                    n = n.children[-1]
+                if n.type in ('async_funcdef', 'async_stmt'):
+                    n = n.children[-1]
+                if n.type in ('classdef', 'funcdef'):
+                    suite_node = n.children[-1]
+                else:
+                    suite_node = c[-1]
+
+                if suite_node.type in ('error_leaf', 'error_node'):
+                    break
 
             new_nodes.append(node)
 
+        # Pop error nodes at the end from the list
+        if new_nodes:
+            while new_nodes:
+                last_node = new_nodes[-1]
+                if (last_node.type in ('error_leaf', 'error_node')
+                        or _is_flow_node(new_nodes[-1])):
+                    # Error leafs/nodes don't have a defined start/end. Error
+                    # nodes might not end with a newline (e.g. if there's an
+                    # open `(`). Therefore ignore all of them unless they are
+                    # succeeded with valid parser state.
+                    # If we copy flows at the end, they might be continued
+                    # after the copy limit (in the new parser).
+                    # In this while loop we try to remove until we find a newline.
+                    new_prefix = ''
+                    new_nodes.pop()
+                    while new_nodes:
+                        last_node = new_nodes[-1]
+                        if last_node.get_last_leaf().type == 'newline':
+                            break
+                        new_nodes.pop()
+                    continue
+                if len(new_nodes) > 1 and new_nodes[-2].type == 'error_node':
+                    # The problem here is that Parso error recovery sometimes
+                    # influences nodes before this node.
+                    # Since the new last node is an error node this will get
+                    # cleaned up in the next while iteration.
+                    new_nodes.pop()
+                    continue
+                break
+
         if not new_nodes:
-            return [], working_stack, prefix
+            return [], working_stack, prefix, added_indents
 
         tos = working_stack[-1]
         last_node = new_nodes[-1]
         had_valid_suite_last = False
+        # Pop incomplete suites from the list
         if _func_or_class_has_suite(last_node):
             suite = last_node
             while suite.type != 'suite':
                 suite = suite.children[-1]
 
-            suite_tos = _NodesTreeNode(suite)
+            indent = _get_suite_indentation(suite)
+            added_indents.append(indent)
+
+            suite_tos = _NodesTreeNode(suite, indentation=_get_indentation(last_node))
             # Don't need to pass line_offset here, it's already done by the
             # parent.
-            suite_nodes, new_working_stack, new_prefix = self._copy_nodes(
-                working_stack + [suite_tos], suite.children, until_line, line_offset
+            suite_nodes, new_working_stack, new_prefix, ai = self._copy_nodes(
+                working_stack + [suite_tos], suite.children, until_line, line_offset,
+                is_nested=True,
             )
+            added_indents += ai
             if len(suite_nodes) < 2:
                 # A suite only with newline is not valid.
                 new_nodes.pop()
@@ -650,25 +833,6 @@ class _NodesTree(object):
                 working_stack = new_working_stack
                 had_valid_suite_last = True
 
-        if new_nodes:
-            last_node = new_nodes[-1]
-            if (last_node.type in ('error_leaf', 'error_node') or
-                    _is_flow_node(new_nodes[-1])):
-                # Error leafs/nodes don't have a defined start/end. Error
-                # nodes might not end with a newline (e.g. if there's an
-                # open `(`). Therefore ignore all of them unless they are
-                # succeeded with valid parser state.
-                # If we copy flows at the end, they might be continued
-                # after the copy limit (in the new parser).
-                # In this while loop we try to remove until we find a newline.
-                new_prefix = ''
-                new_nodes.pop()
-                while new_nodes:
-                    last_node = new_nodes[-1]
-                    if last_node.get_last_leaf().type == 'newline':
-                        break
-                    new_nodes.pop()
-
         if new_nodes:
             if not _ends_with_newline(new_nodes[-1].get_last_leaf()) and not had_valid_suite_last:
                 p = new_nodes[-1].get_next_leaf().prefix
@@ -688,11 +852,13 @@ class _NodesTree(object):
                 assert last_line_offset_leaf == ':'
             else:
                 last_line_offset_leaf = new_nodes[-1].get_last_leaf()
-            tos.add_tree_nodes(prefix, new_nodes, line_offset, last_line_offset_leaf)
+            tos.add_tree_nodes(
+                prefix, new_nodes, line_offset, last_line_offset_leaf,
+            )
             prefix = new_prefix
             self._prefix_remainder = ''
 
-        return new_nodes, working_stack, prefix
+        return new_nodes, working_stack, prefix, added_indents
 
     def close(self):
         self._base_node.finish()
@@ -708,6 +874,8 @@ class _NodesTree(object):
         lines = split_lines(self.prefix)
         assert len(lines) > 0
         if len(lines) == 1:
+            if lines[0].startswith(BOM_UTF8_STRING) and end_pos == [1, 0]:
+                end_pos[1] -= 1
             end_pos[1] += len(lines[0])
         else:
             end_pos[0] += len(lines) - 1

parso/python/errors.py

@@ -6,6 +6,7 @@ from contextlib import contextmanager
 
 from parso.normalizer import Normalizer, NormalizerConfig, Issue, Rule
 from parso.python.tree import search_ancestor
+from parso.python.tokenize import _get_token_collection
 
 _BLOCK_STMTS = ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt')
 _STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist')
@@ -13,11 +14,84 @@ _STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist')
 _MAX_BLOCK_SIZE = 20
 _MAX_INDENT_COUNT = 100
 ALLOWED_FUTURES = (
-    'all_feature_names', 'nested_scopes', 'generators', 'division',
-    'absolute_import', 'with_statement', 'print_function', 'unicode_literals',
+    'nested_scopes', 'generators', 'division', 'absolute_import',
+    'with_statement', 'print_function', 'unicode_literals',
 )
 _COMP_FOR_TYPES = ('comp_for', 'sync_comp_for')
 
+def _get_rhs_name(node, version):
+    type_ = node.type
+    if type_ == "lambdef":
+        return "lambda"
+    elif type_ == "atom":
+        comprehension = _get_comprehension_type(node)
+        first, second = node.children[:2]
+        if comprehension is not None:
+            return comprehension
+        elif second.type == "dictorsetmaker":
+            if version < (3, 8):
+                return "literal"
+            else:
+                if second.children[1] == ":" or second.children[0] == "**":
+                    return "dict display"
+                else:
+                    return "set display"
+        elif (
+            first == "("
+            and (second == ")"
+            or (len(node.children) == 3 and node.children[1].type == "testlist_comp"))
+        ):
+            return "tuple"
+        elif first == "(":
+            return _get_rhs_name(_remove_parens(node), version=version)
+        elif first == "[":
+            return "list"
+        elif first == "{" and second == "}":
+            return "dict display"
+        elif first == "{" and len(node.children) > 2:
+            return "set display"
+    elif type_ == "keyword":
+        if "yield" in node.value:
+            return "yield expression"
+        if version < (3, 8):
+            return "keyword"
+        else:
+            return str(node.value)
+    elif type_ == "operator" and node.value == "...":
+        return "Ellipsis"
+    elif type_ == "comparison":
+        return "comparison"
+    elif type_ in ("string", "number", "strings"):
+        return "literal"
+    elif type_ == "yield_expr":
+        return "yield expression"
+    elif type_ == "test":
+        return "conditional expression"
+    elif type_ in ("atom_expr", "power"):
+        if node.children[0] == "await":
+            return "await expression"
+        elif node.children[-1].type == "trailer":
+            trailer = node.children[-1]
+            if trailer.children[0] == "(":
+                return "function call"
+            elif trailer.children[0] == "[":
+                return "subscript"
+            elif trailer.children[0] == ".":
+                return "attribute"
+    elif (
+        ("expr" in type_
+        and "star_expr" not in type_)  # is a substring
+        or "_test" in type_
+        or type_ in ("term", "factor")
+    ):
+        return "operator"
+    elif type_ == "star_expr":
+        return "starred"
+    elif type_ == "testlist_star_expr":
+        return "tuple"
+    elif type_ == "fstring":
+        return "f-string expression"
+    return type_ # shouldn't reach here
 
 def _iter_stmts(scope):
     """
@@ -136,6 +210,21 @@ def _get_for_stmt_definition_exprs(for_stmt):
     return list(_iter_definition_exprs_from_lists(exprlist))
 
 
+def _is_argument_comprehension(argument):
+    return argument.children[1].type in _COMP_FOR_TYPES
+
+
+def _any_fstring_error(version, node):
+    if version < (3, 9) or node is None:
+        return False
+    if node.type == "error_node":
+        return any(child.type == "fstring_start" for child in node.children)
+    elif node.type == "fstring":
+        return True
+    else:
+        return search_ancestor(node, "fstring")
+
+
 class _Context(object):
     def __init__(self, node, add_syntax_error, parent_context=None):
         self.node = node
@@ -333,6 +422,11 @@ class ErrorFinder(Normalizer):
                     match = re.match('\\w{,2}("{1,3}|\'{1,3})', leaf.value)
                     if match is None:
                         message = 'invalid syntax'
+                        if (
+                            self.version >= (3, 9)
+                            and leaf.value in _get_token_collection(self.version).always_break_tokens
+                        ):
+                            message = "f-string: " + message
                     else:
                         if len(match.group(1)) == 1:
                             message = 'EOL while scanning string literal'
@@ -371,8 +465,8 @@ class ErrorFinder(Normalizer):
 class IndentationRule(Rule):
     code = 903
 
-    def _get_message(self, message):
-        message = super(IndentationRule, self)._get_message(message)
+    def _get_message(self, message, node):
+        message = super(IndentationRule, self)._get_message(message, node)
         return "IndentationError: " + message
 
 
@@ -396,21 +490,34 @@ class ErrorFinderConfig(NormalizerConfig):
 class SyntaxRule(Rule):
     code = 901
 
-    def _get_message(self, message):
-        message = super(SyntaxRule, self)._get_message(message)
+    def _get_message(self, message, node):
+        message = super(SyntaxRule, self)._get_message(message, node)
+        if (
+            "f-string" not in message
+            and _any_fstring_error(self._normalizer.version, node)
+        ):
+            message = "f-string: " + message
         return "SyntaxError: " + message
 
 
 @ErrorFinder.register_rule(type='error_node')
 class _InvalidSyntaxRule(SyntaxRule):
     message = "invalid syntax"
+    fstring_message = "f-string: invalid syntax"
 
     def get_node(self, node):
         return node.get_next_leaf()
 
     def is_issue(self, node):
-        # Error leafs will be added later as an error.
-        return node.get_next_leaf().type != 'error_leaf'
+        error = node.get_next_leaf().type != 'error_leaf'
+        if (
+            error
+            and _any_fstring_error(self._normalizer.version, node)
+        ):
+            self.add_issue(node, message=self.fstring_message)
+        else:
+            # Error leafs will be added later as an error.
+            return error
 
 
 @ErrorFinder.register_rule(value='await')
@@ -449,7 +556,11 @@ class _ContinueChecks(SyntaxRule):
                 in_loop = True
             if block.type == 'try_stmt':
                 last_block = block.children[-3]
-                if last_block == 'finally' and leaf.start_pos > last_block.start_pos:
+                if (
+                    last_block == "finally"
+                    and leaf.start_pos > last_block.start_pos
+                    and self._normalizer.version < (3, 8)
+                ):
                     self.add_issue(leaf, message=self.message_in_finally)
                     return False  # Error already added
         if not in_loop:
@@ -622,26 +733,24 @@ class _FutureImportRule(SyntaxRule):
                 allowed_futures = list(ALLOWED_FUTURES)
                 if self._normalizer.version >= (3, 5):
                     allowed_futures.append('generator_stop')
-
+                if self._normalizer.version >= (3, 7):
+                    allowed_futures.append('annotations')
                 if name == 'braces':
                     self.add_issue(node, message="not a chance")
                 elif name == 'barry_as_FLUFL':
                     m = "Seriously I'm not implementing this :) ~ Dave"
                     self.add_issue(node, message=m)
-                elif name not in ALLOWED_FUTURES:
+                elif name not in allowed_futures:
                     message = "future feature %s is not defined" % name
                     self.add_issue(node, message=message)
 
 
 @ErrorFinder.register_rule(type='star_expr')
 class _StarExprRule(SyntaxRule):
-    message = "starred assignment target must be in a list or tuple"
     message_iterable_unpacking = "iterable unpacking cannot be used in comprehension"
     message_assignment = "can use starred expression only as assignment target"
 
     def is_issue(self, node):
-        if node.parent.type not in _STAR_EXPR_PARENTS:
-            return True
         if node.parent.type == 'testlist_comp':
             # [*[] for a in [1]]
             if node.parent.children[1].type in _COMP_FOR_TYPES:
@@ -665,7 +774,10 @@ class _StarExprRule(SyntaxRule):
 class _StarExprParentRule(SyntaxRule):
     def is_issue(self, node):
         if node.parent.type == 'del_stmt':
-            self.add_issue(node.parent, message="can't use starred expression here")
+            if self._normalizer.version >= (3, 9):
+                self.add_issue(node.parent, message="cannot delete starred")
+            else:
+                self.add_issue(node.parent, message="can't use starred expression here")
         else:
             def is_definition(node, ancestor):
                 if ancestor is None:
@@ -684,7 +796,10 @@ class _StarExprParentRule(SyntaxRule):
                 args = [c for c in node.children if c != ',']
                 starred = [c for c in args if c.type == 'star_expr']
                 if len(starred) > 1:
-                    message = "two starred expressions in assignment"
+                    if self._normalizer.version < (3, 9):
+                        message = "two starred expressions in assignment"
+                    else:
+                        message = "multiple starred expressions in assignment"
                     self.add_issue(starred[1], message=message)
                 elif starred:
                     count = args.index(starred[0])
@@ -734,6 +849,9 @@ class _AnnotatorRule(SyntaxRule):
 class _ArgumentRule(SyntaxRule):
     def is_issue(self, node):
         first = node.children[0]
+        if self._normalizer.version < (3, 8):
+            # a((b)=c) is valid in <3.8
+            first = _remove_parens(first)
         if node.children[1] == '=' and first.type != 'name':
             if first.type == 'lambdef':
                 # f(lambda: 1=1)
@@ -749,6 +867,9 @@ class _ArgumentRule(SyntaxRule):
                     message = 'expression cannot contain assignment, perhaps you meant "=="?'
             self.add_issue(first, message=message)
 
+        if _is_argument_comprehension(node) and node.parent.type == 'classdef':
+            self.add_issue(node, message='invalid syntax')
+
 
 @ErrorFinder.register_rule(type='nonlocal_stmt')
 class _NonlocalModuleLevelRule(SyntaxRule):
@@ -768,59 +889,60 @@ class _ArglistRule(SyntaxRule):
             return "Generator expression must be parenthesized"
 
     def is_issue(self, node):
-        first_arg = node.children[0]
-        if first_arg.type == 'argument' \
-                and first_arg.children[1].type in _COMP_FOR_TYPES:
-            # e.g. foo(x for x in [], b)
-            return len(node.children) >= 2
-        else:
-            arg_set = set()
-            kw_only = False
-            kw_unpacking_only = False
-            is_old_starred = False
-            # In python 3 this would be a bit easier (stars are part of
-            # argument), but we have to understand both.
-            for argument in node.children:
-                if argument == ',':
-                    continue
+        arg_set = set()
+        kw_only = False
+        kw_unpacking_only = False
+        is_old_starred = False
+        # In python 3 this would be a bit easier (stars are part of
+        # argument), but we have to understand both.
+        for argument in node.children:
+            if argument == ',':
+                continue
 
-                if argument in ('*', '**'):
-                    # Python < 3.5 has the order engraved in the grammar
-                    # file.  No need to do anything here.
-                    is_old_starred = True
-                    continue
-                if is_old_starred:
-                    is_old_starred = False
-                    continue
+            if argument in ('*', '**'):
+                # Python < 3.5 has the order engraved in the grammar
+                # file.  No need to do anything here.
+                is_old_starred = True
+                continue
+            if is_old_starred:
+                is_old_starred = False
+                continue
 
-                if argument.type == 'argument':
-                    first = argument.children[0]
-                    if first in ('*', '**'):
-                        if first == '*':
-                            if kw_unpacking_only:
-                                # foo(**kwargs, *args)
-                                message = "iterable argument unpacking " \
-                                          "follows keyword argument unpacking"
-                                self.add_issue(argument, message=message)
+            if argument.type == 'argument':
+                first = argument.children[0]
+                if _is_argument_comprehension(argument) and len(node.children) >= 2:
+                    # a(a, b for b in c)
+                    return True
+
+                if first in ('*', '**'):
+                    if first == '*':
+                        if kw_unpacking_only:
+                            # foo(**kwargs, *args)
+                            message = "iterable argument unpacking " \
+                                      "follows keyword argument unpacking"
+                            self.add_issue(argument, message=message)
+                    else:
+                        kw_unpacking_only = True
+                else:  # Is a keyword argument.
+                    kw_only = True
+                    if first.type == 'name':
+                        if first.value in arg_set:
+                            # f(x=1, x=2)
+                            message = "keyword argument repeated"
+                            if self._normalizer.version >= (3, 9):
+                                message += ": {}".format(first.value)
+                            self.add_issue(first, message=message)
                         else:
-                            kw_unpacking_only = True
-                    else:  # Is a keyword argument.
-                        kw_only = True
-                        if first.type == 'name':
-                            if first.value in arg_set:
-                                # f(x=1, x=2)
-                                self.add_issue(first, message="keyword argument repeated")
-                            else:
-                                arg_set.add(first.value)
-                else:
-                    if kw_unpacking_only:
-                        # f(**x, y)
-                        message = "positional argument follows keyword argument unpacking"
-                        self.add_issue(argument, message=message)
-                    elif kw_only:
-                        # f(x=2, y)
-                        message = "positional argument follows keyword argument"
-                        self.add_issue(argument, message=message)
+                            arg_set.add(first.value)
+            else:
+                if kw_unpacking_only:
+                    # f(**x, y)
+                    message = "positional argument follows keyword argument unpacking"
+                    self.add_issue(argument, message=message)
+                elif kw_only:
+                    # f(x=2, y)
+                    message = "positional argument follows keyword argument"
+                    self.add_issue(argument, message=message)
 
 
 @ErrorFinder.register_rule(type='parameters')
@@ -898,7 +1020,7 @@ class _FStringRule(SyntaxRule):
 
 
 class _CheckAssignmentRule(SyntaxRule):
-    def _check_assignment(self, node, is_deletion=False, is_namedexpr=False):
+    def _check_assignment(self, node, is_deletion=False, is_namedexpr=False, is_aug_assign=False):
         error = None
         type_ = node.type
         if type_ == 'lambdef':
@@ -915,6 +1037,16 @@ class _CheckAssignmentRule(SyntaxRule):
                             error = 'dict display'
                         else:
                             error = 'set display'
+                elif first == "{" and second == "}":
+                    if self._normalizer.version < (3, 8):
+                        error = 'literal'
+                    else:
+                        error = "dict display"
+                elif first == "{" and len(node.children) > 2:
+                    if self._normalizer.version < (3, 8):
+                        error = 'literal'
+                    else:
+                        error = "set display"
                 elif first in ('(', '['):
                     if second.type == 'yield_expr':
                         error = 'yield expression'
@@ -930,11 +1062,13 @@ class _CheckAssignmentRule(SyntaxRule):
                         # This is not a comprehension, they were handled
                         # further above.
                         for child in second.children[::2]:
-                            self._check_assignment(child, is_deletion, is_namedexpr)
+                            self._check_assignment(child, is_deletion, is_namedexpr, is_aug_assign)
                     else:  # Everything handled, must be useless brackets.
-                        self._check_assignment(second, is_deletion, is_namedexpr)
+                        self._check_assignment(second, is_deletion, is_namedexpr, is_aug_assign)
         elif type_ == 'keyword':
-            if self._normalizer.version < (3, 8):
+            if node.value == "yield":
+                error = "yield expression"
+            elif self._normalizer.version < (3, 8):
                 error = 'keyword'
             else:
                 error = str(node.value)
@@ -966,19 +1100,32 @@ class _CheckAssignmentRule(SyntaxRule):
                     error = 'subscript'
                 elif is_namedexpr and trailer.children[0] == '.':
                     error = 'attribute'
+        elif type_ == "fstring":
+            if self._normalizer.version < (3, 8):
+                error = 'literal'
+            else:
+                error = "f-string expression"
         elif type_ in ('testlist_star_expr', 'exprlist', 'testlist'):
             for child in node.children[::2]:
-                self._check_assignment(child, is_deletion, is_namedexpr)
+                self._check_assignment(child, is_deletion, is_namedexpr, is_aug_assign)
         elif ('expr' in type_ and type_ != 'star_expr'  # is a substring
               or '_test' in type_
               or type_ in ('term', 'factor')):
             error = 'operator'
+        elif type_ == "star_expr":
+            if is_deletion:
+                if self._normalizer.version >= (3, 9):
+                    error = "starred"
+                else:
+                    self.add_issue(node, message="can't use starred expression here")
+            elif not search_ancestor(node, *_STAR_EXPR_PARENTS) and not is_aug_assign:
+                self.add_issue(node, message="starred assignment target must be in a list or tuple")
+
+            self._check_assignment(node.children[1])
 
         if error is not None:
             if is_namedexpr:
-                # c.f. CPython bpo-39176, should be changed in next release
-                # message = 'cannot use assignment expressions with %s' % error
-                message = 'cannot use named assignment with %s' % error
+                message = 'cannot use assignment expressions with %s' % error
             else:
                 cannot = "can't" if self._normalizer.version < (3, 8) else "cannot"
                 message = ' '.join([cannot, "delete" if is_deletion else "assign to", error])
@@ -1001,15 +1148,35 @@ class _CompForRule(_CheckAssignmentRule):
 @ErrorFinder.register_rule(type='expr_stmt')
 class _ExprStmtRule(_CheckAssignmentRule):
     message = "illegal expression for augmented assignment"
-
+    extended_message = "'{target}' is an " + message
     def is_issue(self, node):
-        for before_equal in node.children[:-2:2]:
-            self._check_assignment(before_equal)
-
         augassign = node.children[1]
-        if augassign != '=' and augassign.type != 'annassign':  # Is augassign.
-            return node.children[0].type in ('testlist_star_expr', 'atom', 'testlist')
+        is_aug_assign = augassign != '=' and augassign.type != 'annassign'
 
+        if self._normalizer.version <= (3, 8) or not is_aug_assign:
+            for before_equal in node.children[:-2:2]:
+                self._check_assignment(before_equal, is_aug_assign=is_aug_assign)
+
+        if is_aug_assign:
+            target = _remove_parens(node.children[0])
+            # a, a[b], a.b
+
+            if target.type == "name" or (
+                target.type in ("atom_expr", "power")
+                and target.children[1].type == "trailer"
+                and target.children[-1].children[0] != "("
+            ):
+                return False
+
+            if self._normalizer.version <= (3, 8):
+                return True
+            else:
+                self.add_issue(
+                    node,
+                    message=self.extended_message.format(
+                        target=_get_rhs_name(node.children[0], self._normalizer.version)
+                    ),
+                )
 
 @ErrorFinder.register_rule(type='with_item')
 class _WithItemRule(_CheckAssignmentRule):

parso/python/grammar27.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() and input() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -30,7 +30,7 @@ varargslist: ((fpdef ['=' test] ',')*
 fpdef: NAME | '(' fplist ')'
 fplist: fpdef (',' fpdef)* [',']
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | exec_stmt | assert_stmt)

parso/python/grammar310.txt

@@ -0,0 +1,171 @@
+# Grammar for Python
+
+# NOTE WELL: You should also follow all the steps listed at
+# https://devguide.python.org/grammar/
+
+# Start symbols for the grammar:
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+file_input: stmt* ENDMARKER
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' namedexpr_test NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+async_funcdef: 'async' funcdef
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+
+parameters: '(' [typedargslist] ')'
+typedargslist: (
+  (tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [',' [ tfpdef ['=' test] (
+        ',' tfpdef ['=' test])* ([',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]])
+  | '*' [tfpdef] (',' tfpdef ['=' test])* ([',' ['**' tfpdef [',']]])
+  | '**' tfpdef [',']]] )
+|  (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]]
+  | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+  | '**' tfpdef [','])
+)
+tfpdef: NAME [':' test]
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']
+)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt | NEWLINE
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal and annotated assignments, additional restrictions enforced by the interpreter
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist_star_expr]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test ['from' test]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test ['as' NAME]]
+suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
+
+namedexpr_test: test [':=' test]
+test: or_test ['if' or_test 'else' test] | lambdef
+test_nocond: or_test | lambdef_nocond
+lambdef: 'lambda' [varargslist] ':' test
+lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401 (which really works :-)
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'@'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
+testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test | star_expr)
+                   (comp_for | (',' (test | star_expr))* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: argument (',' argument)*  [',']
+
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguments are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test ':=' test |
+            test '=' test |
+            '**' test |
+            '*' test )
+
+comp_iter: comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
+comp_if: 'if' test_nocond [comp_iter]
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist_star_expr
+
+strings: (STRING | fstring)+
+fstring: FSTRING_START fstring_content* FSTRING_END
+fstring_content: FSTRING_STRING | fstring_expr
+fstring_conversion: '!' NAME
+fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_format_spec: ':' fstring_content*

parso/python/grammar33.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -33,7 +33,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
      |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)

parso/python/grammar34.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -33,7 +33,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
      |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)

parso/python/grammar35.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -38,7 +38,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
      |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)

parso/python/grammar36.txt

@@ -9,7 +9,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
@@ -35,7 +35,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
 )
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)

parso/python/grammar37.txt

@@ -9,7 +9,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
@@ -33,7 +33,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
 )
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)

parso/python/grammar38.txt

@@ -9,7 +9,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -46,13 +46,13 @@ varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef [
 )
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
 expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist_star_expr))*)
-annassign: ':' test ['=' test]
+annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
 testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
 augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')

parso/python/grammar39.txt

@@ -9,10 +9,10 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
-decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
+decorator: '@' namedexpr_test NEWLINE
 decorators: decorator+
 decorated: decorators (classdef | funcdef | async_funcdef)
 
@@ -46,13 +46,13 @@ varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef [
 )
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
 expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist_star_expr))*)
-annassign: ':' test ['=' test]
+annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
 testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
 augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')

parso/python/parser.py

@@ -126,10 +126,10 @@ class Parser(BaseParser):
 
         if self._start_nonterminal == 'file_input' and \
                 (token.type == PythonTokenTypes.ENDMARKER
-                 or token.type == DEDENT and '\n' not in last_leaf.value
-                 and '\r' not in last_leaf.value):
+                 or token.type == DEDENT and not last_leaf.value.endswith('\n')
+                 and not last_leaf.value.endswith('\r')):
             # In Python statements need to end with a newline. But since it's
-            # possible (and valid in Python ) that there's no newline at the
+            # possible (and valid in Python) that there's no newline at the
             # end of a file, we have to recover even if the user doesn't want
             # error recovery.
             if self.stack[-1].dfa.from_rule == 'simple_stmt':
@@ -208,6 +208,7 @@ class Parser(BaseParser):
                 o = self._omit_dedent_list
                 if o and o[-1] == self._indent_counter:
                     o.pop()
+                    self._indent_counter -= 1
                     continue
 
                 self._indent_counter -= 1

parso/python/tokenize.py

@@ -12,7 +12,6 @@ memory optimizations here.
 from __future__ import absolute_import
 
 import sys
-import string
 import re
 from collections import namedtuple
 import itertools as _itertools
@@ -218,10 +217,10 @@ def _create_token_collection(version_info):
     Funny = group(Operator, Bracket, Special)
 
     # First (or only) line of ' or " string.
-    ContStr = group(StringPrefix + r"'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*" +
-                    group("'", r'\\(?:\r\n?|\n)'),
-                    StringPrefix + r'"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*' +
-                    group('"', r'\\(?:\r\n?|\n)'))
+    ContStr = group(StringPrefix + r"'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*"
+                    + group("'", r'\\(?:\r\n?|\n)'),
+                    StringPrefix + r'"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*'
+                    + group('"', r'\\(?:\r\n?|\n)'))
     pseudo_extra_pool = [Comment, Triple]
     all_quotes = '"', "'", '"""', "'''"
     if fstring_prefixes:
@@ -258,11 +257,14 @@ def _create_token_collection(version_info):
             fstring_pattern_map[t + quote] = quote
 
     ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except',
-                           'finally', 'while', 'with', 'return')
+                           'finally', 'while', 'with', 'return', 'continue',
+                           'break', 'del', 'pass', 'global', 'assert')
+    if version_info >= (3, 5):
+        ALWAYS_BREAK_TOKENS += ('nonlocal', )
     pseudo_token_compiled = _compile(PseudoToken)
     return TokenCollection(
         pseudo_token_compiled, single_quoted, triple_quoted, endpats,
-        whitespace, fstring_pattern_map, ALWAYS_BREAK_TOKENS
+        whitespace, fstring_pattern_map, set(ALWAYS_BREAK_TOKENS)
     )
 
 
@@ -311,7 +313,7 @@ class FStringNode(object):
         return not self.is_in_expr() and self.format_spec_count
 
 
-def _close_fstring_if_necessary(fstring_stack, string, start_pos, additional_prefix):
+def _close_fstring_if_necessary(fstring_stack, string, line_nr, column, additional_prefix):
     for fstring_stack_index, node in enumerate(fstring_stack):
         lstripped_string = string.lstrip()
         len_lstrip = len(string) - len(lstripped_string)
@@ -319,7 +321,7 @@ def _close_fstring_if_necessary(fstring_stack, string, start_pos, additional_pre
             token = PythonToken(
                 FSTRING_END,
                 node.quote,
-                start_pos,
+                (line_nr, column + len_lstrip),
                 prefix=additional_prefix+string[:len_lstrip],
             )
             additional_prefix = ''
@@ -381,13 +383,14 @@ def _print_tokens(func):
     """
     def wrapper(*args, **kwargs):
         for token in func(*args, **kwargs):
+            print(token)  # This print is intentional for debugging!
             yield token
 
     return wrapper
 
 
 # @_print_tokens
-def tokenize_lines(lines, version_info, start_pos=(1, 0)):
+def tokenize_lines(lines, version_info, start_pos=(1, 0), indents=None, is_first_token=True):
     """
     A heavily modified Python standard library tokenizer.
 
@@ -398,17 +401,19 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
     def dedent_if_necessary(start):
         while start < indents[-1]:
             if start > indents[-2]:
-                yield PythonToken(ERROR_DEDENT, '', (lnum, 0), '')
+                yield PythonToken(ERROR_DEDENT, '', (lnum, start), '')
+                indents[-1] = start
                 break
-            yield PythonToken(DEDENT, '', spos, '')
             indents.pop()
+            yield PythonToken(DEDENT, '', spos, '')
 
     pseudo_token, single_quoted, triple_quoted, endpats, whitespace, \
         fstring_pattern_map, always_break_tokens, = \
         _get_token_collection(version_info)
     paren_level = 0  # count parentheses
-    indents = [0]
-    max = 0
+    if indents is None:
+        indents = [0]
+    max_ = 0
     numchars = '0123456789'
     contstr = ''
     contline = None
@@ -419,25 +424,24 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
     new_line = True
     prefix = ''  # Should never be required, but here for safety
     additional_prefix = ''
-    first = True
     lnum = start_pos[0] - 1
     fstring_stack = []
     for line in lines:  # loop over lines in stream
         lnum += 1
         pos = 0
-        max = len(line)
-        if first:
+        max_ = len(line)
+        if is_first_token:
             if line.startswith(BOM_UTF8_STRING):
                 additional_prefix = BOM_UTF8_STRING
                 line = line[1:]
-                max = len(line)
+                max_ = len(line)
 
             # Fake that the part before was already parsed.
             line = '^' * start_pos[1] + line
             pos = start_pos[1]
-            max += start_pos[1]
+            max_ += start_pos[1]
 
-            first = False
+            is_first_token = False
 
         if contstr:                                         # continued string
             endmatch = endprog.match(line)
@@ -453,7 +457,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 contline = contline + line
                 continue
 
-        while pos < max:
+        while pos < max_:
             if fstring_stack:
                 tos = fstring_stack[-1]
                 if not tos.is_in_expr():
@@ -468,14 +472,15 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                         )
                         tos.previous_lines = ''
                         continue
-                    if pos == max:
+                    if pos == max_:
                         break
 
                 rest = line[pos:]
                 fstring_end_token, additional_prefix, quote_length = _close_fstring_if_necessary(
                     fstring_stack,
                     rest,
-                    (lnum, pos),
+                    lnum,
+                    pos,
                     additional_prefix,
                 )
                 pos += quote_length
@@ -496,9 +501,39 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 pseudomatch = pseudo_token.match(string_line, pos)
             else:
                 pseudomatch = pseudo_token.match(line, pos)
+
+            if pseudomatch:
+                prefix = additional_prefix + pseudomatch.group(1)
+                additional_prefix = ''
+                start, pos = pseudomatch.span(2)
+                spos = (lnum, start)
+                token = pseudomatch.group(2)
+                if token == '':
+                    assert prefix
+                    additional_prefix = prefix
+                    # This means that we have a line with whitespace/comments at
+                    # the end, which just results in an endmarker.
+                    break
+                initial = token[0]
+            else:
+                match = whitespace.match(line, pos)
+                initial = line[match.end()]
+                start = match.end()
+                spos = (lnum, start)
+
+            if new_line and initial not in '\r\n#' and (initial != '\\' or pseudomatch is None):
+                new_line = False
+                if paren_level == 0 and not fstring_stack:
+                    indent_start = start
+                    if indent_start > indents[-1]:
+                        yield PythonToken(INDENT, '', spos, '')
+                        indents.append(indent_start)
+                    for t in dedent_if_necessary(indent_start):
+                        yield t
+
             if not pseudomatch:  # scan for tokens
                 match = whitespace.match(line, pos)
-                if pos == 0:
+                if new_line and paren_level == 0 and not fstring_stack:
                     for t in dedent_if_necessary(match.end()):
                         yield t
                 pos = match.end()
@@ -511,50 +546,18 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 pos += 1
                 continue
 
-            prefix = additional_prefix + pseudomatch.group(1)
-            additional_prefix = ''
-            start, pos = pseudomatch.span(2)
-            spos = (lnum, start)
-            token = pseudomatch.group(2)
-            if token == '':
-                assert prefix
-                additional_prefix = prefix
-                # This means that we have a line with whitespace/comments at
-                # the end, which just results in an endmarker.
-                break
-            initial = token[0]
-
-            if new_line and initial not in '\r\n\\#':
-                new_line = False
-                if paren_level == 0 and not fstring_stack:
-                    i = 0
-                    indent_start = start
-                    while line[i] == '\f':
-                        i += 1
-                        # TODO don't we need to change spos as well?
-                        indent_start -= 1
-                    if indent_start > indents[-1]:
-                        yield PythonToken(INDENT, '', spos, '')
-                        indents.append(indent_start)
-                    for t in dedent_if_necessary(indent_start):
-                        yield t
-
-            if (initial in numchars or                      # ordinary number
-                    (initial == '.' and token != '.' and token != '...')):
+            if (initial in numchars                      # ordinary number
+                    or (initial == '.' and token != '.' and token != '...')):
                 yield PythonToken(NUMBER, token, spos, prefix)
             elif pseudomatch.group(3) is not None:            # ordinary name
-                if token in always_break_tokens:
+                if token in always_break_tokens and (fstring_stack or paren_level):
                     fstring_stack[:] = []
                     paren_level = 0
                     # We only want to dedent if the token is on a new line.
-                    if re.match(r'[ \f\t]*$', line[:start]):
-                        while True:
-                            indent = indents.pop()
-                            if indent > start:
-                                yield PythonToken(DEDENT, '', spos, '')
-                            else:
-                                indents.append(indent)
-                                break
+                    m = re.match(r'[ \f\t]*$', line[:start])
+                    if m is not None:
+                        for t in dedent_if_necessary(m.end()):
+                            yield t
                 if is_identifier(token):
                     yield PythonToken(NAME, token, spos, prefix)
                 else:
@@ -587,7 +590,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                     token = line[start:pos]
                     yield PythonToken(STRING, token, spos, prefix)
                 else:
-                    contstr_start = (lnum, start)           # multiple lines
+                    contstr_start = spos                    # multiple lines
                     contstr = line[start:]
                     contline = line
                     break
@@ -649,10 +652,22 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
         if contstr.endswith('\n') or contstr.endswith('\r'):
             new_line = True
 
-    end_pos = lnum, max
+    if fstring_stack:
+        tos = fstring_stack[-1]
+        if tos.previous_lines:
+            yield PythonToken(
+                FSTRING_STRING, tos.previous_lines,
+                tos.last_string_start_pos,
+                # Never has a prefix because it can start anywhere and
+                # include whitespace.
+                prefix=''
+            )
+
+    end_pos = lnum, max_
     # As the last position we just take the maximally possible position. We
     # remove -1 for the last new line.
     for indent in indents[1:]:
+        indents.pop()
         yield PythonToken(DEDENT, '', end_pos, '')
     yield PythonToken(ENDMARKER, '', end_pos, additional_prefix)
 

parso/python/tree.py

@@ -1081,7 +1081,13 @@ class ExprStmt(PythonBaseNode, DocstringMixin):
 
     def get_rhs(self):
         """Returns the right-hand-side of the equals."""
-        return self.children[-1]
+        node = self.children[-1]
+        if node.type == 'annassign':
+            if len(node.children) == 4:
+                node = node.children[3]
+            else:
+                node = node.children[1]
+        return node
 
     def yield_operators(self):
         """

parso/tree.py

@@ -45,8 +45,12 @@ class NodeOrLeaf(object):
         Returns the node immediately following this node in this parent's
         children list. If this node does not have a next sibling, it is None
         """
+        parent = self.parent
+        if parent is None:
+            return None
+
         # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
+        for i, child in enumerate(parent.children):
             if child is self:
                 try:
                     return self.parent.children[i + 1]
@@ -59,8 +63,12 @@ class NodeOrLeaf(object):
         children list. If this node does not have a previous sibling, it is
         None.
         """
+        parent = self.parent
+        if parent is None:
+            return None
+
         # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
+        for i, child in enumerate(parent.children):
             if child is self:
                 if i == 0:
                     return None
@@ -71,6 +79,9 @@ class NodeOrLeaf(object):
         Returns the previous leaf in the parser tree.
         Returns `None` if this is the first element in the parser tree.
         """
+        if self.parent is None:
+            return None
+
         node = self
         while True:
             c = node.parent.children
@@ -94,6 +105,9 @@ class NodeOrLeaf(object):
         Returns the next leaf in the parser tree.
         Returns None if this is the last element in the parser tree.
         """
+        if self.parent is None:
+            return None
+
         node = self
         while True:
             c = node.parent.children
@@ -154,7 +168,7 @@ class NodeOrLeaf(object):
     @abstractmethod
     def get_code(self, include_prefix=True):
         """
-        Returns the code that was input the input for the parser for this node.
+        Returns the code that was the input for the parser for this node.
 
         :param include_prefix: Removes the prefix (whitespace and comments) of
             e.g. a statement.

parso/utils.py

@@ -105,8 +105,17 @@ def python_bytes_to_unicode(source, encoding='utf-8', errors='strict'):
     if not isinstance(encoding, unicode):
         encoding = unicode(encoding, 'utf-8', 'replace')
 
-    # Cast to unicode
-    return unicode(source, encoding, errors)
+    try:
+        # Cast to unicode
+        return unicode(source, encoding, errors)
+    except LookupError:
+        if errors == 'replace':
+            # This is a weird case that can happen if the given encoding is not
+            # a valid encoding. This usually shouldn't happen with provided
+            # encodings, but can happen if somebody uses encoding declarations
+            # like `# coding: foo-8`.
+            return unicode(source, 'utf-8', errors)
+        raise
 
 
 def version_info():
@@ -120,7 +129,7 @@ def version_info():
 
 
 def _parse_version(version):
-    match = re.match(r'(\d+)(?:\.(\d)(?:\.\d+)?)?$', version)
+    match = re.match(r'(\d+)(?:\.(\d{1,2})(?:\.\d+)?)?((a|b|rc)\d)?$', version)
     if match is None:
         raise ValueError('The given version is not in the right format. '
                          'Use something like "3.8" or "3".')

setup.py

@@ -25,7 +25,7 @@ setup(name='parso',
       keywords='python parser parsing',
       long_description=readme,
       packages=find_packages(exclude=['test']),
-      package_data={'parso': ['python/grammar*.txt']},
+      package_data={'parso': ['python/grammar*.txt', 'py.typed', '*.pyi', '**/*.pyi']},
       platforms=['any'],
       python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
       classifiers=[
@@ -44,6 +44,7 @@ setup(name='parso',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Text Editors :: Integrated Development Environments (IDE)',
           'Topic :: Utilities',
+          'Typing :: Typed',
       ],
       extras_require={
           'testing': [

test/failing_examples.py

@@ -52,9 +52,37 @@ FAILING_EXAMPLES = [
     'f(x=2, y)',
     'f(**x, *y)',
     'f(**x, y=3, z)',
+    # augassign
     'a, b += 3',
     '(a, b) += 3',
     '[a, b] += 3',
+    'f() += 1',
+    'lambda x:None+=1',
+    '{} += 1',
+    '{a:b} += 1',
+    '{1} += 1',
+    '{*x} += 1',
+    '(x,) += 1',
+    '(x, y if a else q) += 1',
+    '[] += 1',
+    '[1,2] += 1',
+    '[] += 1',
+    'None += 1',
+    '... += 1',
+    'a > 1 += 1',
+    '"test" += 1',
+    '1 += 1',
+    '1.0 += 1',
+    '(yield) += 1',
+    '(yield from x) += 1',
+    '(x if x else y) += 1',
+    'a() += 1',
+    'a + b += 1',
+    '+a += 1',
+    'a and b += 1',
+    '*a += 1',
+    'a, b += 1',
+    'f"xxx" += 1',
     # All assignment tests
     'lambda a: 1 = 1',
     '[x for x in y] = 1',
@@ -308,6 +336,12 @@ if sys.version_info[:2] <= (3, 4):
         '(*[1], 2)',
     ]
 
+if sys.version_info[:2] >= (3, 7):
+    # This is somehow ok in previous versions.
+    FAILING_EXAMPLES += [
+        'class X(base for base in bases): pass',
+    ]
+
 if sys.version_info[:2] < (3, 8):
     FAILING_EXAMPLES += [
         # Python/compile.c

test/fuzz_diff_parser.py

@@ -50,6 +50,11 @@ def find_python_files_in_tree(file_path):
         yield file_path
         return
     for root, dirnames, filenames in os.walk(file_path):
+        if 'chardet' in root:
+            # Stuff like chardet/langcyrillicmodel.py is just very slow to
+            # parse and machine generated, so ignore those.
+            continue
+
         for name in filenames:
             if name.endswith('.py'):
                 yield os.path.join(root, name)
@@ -102,9 +107,17 @@ class LineCopy:
 
 class FileModification:
     @classmethod
-    def generate(cls, code_lines, change_count):
+    def generate(cls, code_lines, change_count, previous_file_modification=None):
+        if previous_file_modification is not None and random.random() > 0.5:
+            # We want to keep the previous modifications in some cases to make
+            # more complex parser issues visible.
+            code_lines = previous_file_modification.apply(code_lines)
+            added_modifications = previous_file_modification.modification_list
+        else:
+            added_modifications = []
         return cls(
-            list(cls._generate_line_modifications(code_lines, change_count)),
+            added_modifications
+            + list(cls._generate_line_modifications(code_lines, change_count)),
             # work with changed trees more than with normal ones.
             check_original=random.random() > 0.8,
         )
@@ -158,18 +171,18 @@ class FileModification:
             yield l
 
     def __init__(self, modification_list, check_original):
-        self._modification_list = modification_list
+        self.modification_list = modification_list
         self._check_original = check_original
 
-    def _apply(self, code_lines):
+    def apply(self, code_lines):
         changed_lines = list(code_lines)
-        for modification in self._modification_list:
+        for modification in self.modification_list:
             modification.apply(changed_lines)
         return changed_lines
 
     def run(self, grammar, code_lines, print_code):
         code = ''.join(code_lines)
-        modified_lines = self._apply(code_lines)
+        modified_lines = self.apply(code_lines)
         modified_code = ''.join(modified_lines)
 
         if print_code:
@@ -197,7 +210,7 @@ class FileModification:
 class FileTests:
     def __init__(self, file_path, test_count, change_count):
         self._path = file_path
-        with open(file_path) as f:
+        with open(file_path, errors='replace') as f:
             code = f.read()
         self._code_lines = split_lines(code, keepends=True)
         self._test_count = test_count
@@ -228,8 +241,12 @@ class FileTests:
 
     def run(self, grammar, debugger):
         def iterate():
+            fm = None
             for _ in range(self._test_count):
-                fm = FileModification.generate(self._code_lines, self._change_count)
+                fm = FileModification.generate(
+                    self._code_lines, self._change_count,
+                    previous_file_modification=fm
+                )
                 self._file_modifications.append(fm)
                 yield fm
 

test/normalizer_issue_files/allowed_syntax.py

@@ -12,13 +12,6 @@ from .__future__ import absolute_import
 ''r''u''
 b'' BR''
 
-for x in [1]:
-    try:
-        continue  # Only the other continue and pass is an error.
-    finally:
-        #: E901
-        continue
-
 
 for x in [1]:
     break

test/test_cache.py

@@ -2,28 +2,38 @@
 Test all things related to the ``jedi.cache`` module.
 """
 
-from os import unlink
+import os
+import os.path
 
 import pytest
 import time
 
-from parso.cache import _NodeCacheItem, save_module, load_module, \
-    _get_hashed_path, parser_cache, _load_from_file_system, _save_to_file_system
+from parso.cache import (_CACHED_FILE_MAXIMUM_SURVIVAL, _VERSION_TAG,
+                         _get_cache_clear_lock, _get_hashed_path,
+                         _load_from_file_system, _NodeCacheItem,
+                         _remove_cache_and_update_lock, _save_to_file_system,
+                         load_module, parser_cache, try_to_save_module)
+from parso._compatibility import is_pypy, PermissionError
 from parso import load_grammar
 from parso import cache
 from parso import file_io
 from parso import parse
 
+skip_pypy = pytest.mark.skipif(
+    is_pypy,
+    reason="pickling in pypy is slow, since we don't pickle,"
+           "we never go into path of auto-collecting garbage"
+)
+
 
 @pytest.fixture()
-def isolated_jedi_cache(monkeypatch, tmpdir):
-    """
-    Set `jedi.settings.cache_directory` to a temporary directory during test.
-
-    Same as `clean_jedi_cache`, but create the temporary directory for
-    each test case (scope='function').
-    """
-    monkeypatch.setattr(cache, '_default_cache_path', str(tmpdir))
+def isolated_parso_cache(monkeypatch, tmpdir):
+    """Set `parso.cache._default_cache_path` to a temporary directory
+    during the test. """
+    cache_path = str(os.path.join(str(tmpdir), "__parso_cache"))
+    monkeypatch.setattr(cache, '_default_cache_path', cache_path)
+    monkeypatch.setattr(cache, '_get_default_cache_path', lambda *args, **kwargs: cache_path)
+    return cache_path
 
 
 def test_modulepickling_change_cache_dir(tmpdir):
@@ -57,7 +67,7 @@ def load_stored_item(hashed_grammar, path, item, cache_path):
     return item
 
 
-@pytest.mark.usefixtures("isolated_jedi_cache")
+@pytest.mark.usefixtures("isolated_parso_cache")
 def test_modulepickling_simulate_deleted_cache(tmpdir):
     """
     Tests loading from a cache file after it is deleted.
@@ -81,10 +91,10 @@ def test_modulepickling_simulate_deleted_cache(tmpdir):
         pass
     io = file_io.FileIO(path)
 
-    save_module(grammar._hashed, io, module, lines=[])
+    try_to_save_module(grammar._hashed, io, module, lines=[])
     assert load_module(grammar._hashed, io) == module
 
-    unlink(_get_hashed_path(grammar._hashed, path))
+    os.unlink(_get_hashed_path(grammar._hashed, path))
     parser_cache.clear()
 
     cached2 = load_module(grammar._hashed, io)
@@ -139,3 +149,43 @@ def test_cache_last_used_update(diff_cache, use_file_io):
 
     node_cache_item = next(iter(parser_cache.values()))[p]
     assert now < node_cache_item.last_used < time.time()
+
+
+@skip_pypy
+def test_inactive_cache(tmpdir, isolated_parso_cache):
+    parser_cache.clear()
+    test_subjects = "abcdef"
+    for path in test_subjects:
+        parse('somecode', cache=True, path=os.path.join(str(tmpdir), path))
+    raw_cache_path = os.path.join(isolated_parso_cache, _VERSION_TAG)
+    assert os.path.exists(raw_cache_path)
+    paths = os.listdir(raw_cache_path)
+    a_while_ago = time.time() - _CACHED_FILE_MAXIMUM_SURVIVAL
+    old_paths = set()
+    for path in paths[:len(test_subjects) // 2]:  # make certain number of paths old
+        os.utime(os.path.join(raw_cache_path, path), (a_while_ago, a_while_ago))
+        old_paths.add(path)
+    # nothing should be cleared while the lock is on
+    assert os.path.exists(_get_cache_clear_lock().path)
+    _remove_cache_and_update_lock()  # it shouldn't clear anything
+    assert len(os.listdir(raw_cache_path)) == len(test_subjects)
+    assert old_paths.issubset(os.listdir(raw_cache_path))
+
+    os.utime(_get_cache_clear_lock().path, (a_while_ago, a_while_ago))
+    _remove_cache_and_update_lock()
+    assert len(os.listdir(raw_cache_path)) == len(test_subjects) // 2
+    assert not old_paths.intersection(os.listdir(raw_cache_path))
+
+
+@skip_pypy
+def test_permission_error(monkeypatch):
+    def save(*args, **kwargs):
+        was_called[0] = True  # Python 2... Use nonlocal instead
+        raise PermissionError
+
+    was_called = [False]
+
+    monkeypatch.setattr(cache, '_save_to_file_system', save)
+    with pytest.warns(Warning):
+        parse(path=__file__, cache=True, diff_cache=True)
+    assert was_called[0]

test/test_diff_parser.py

@@ -8,7 +8,7 @@ import pytest
 from parso.utils import split_lines
 from parso import cache
 from parso import load_grammar
-from parso.python.diff import DiffParser, _assert_valid_graph
+from parso.python.diff import DiffParser, _assert_valid_graph, _assert_nodes_are_equal
 from parso import parse
 
 ANY = object()
@@ -69,6 +69,9 @@ class Differ(object):
 
         _assert_valid_graph(new_module)
 
+        without_diff_parser_module = parse(code)
+        _assert_nodes_are_equal(new_module, without_diff_parser_module)
+
         error_node = _check_error_leaves_nodes(new_module)
         assert expect_error_leaves == (error_node is not None), error_node
         if parsers is not ANY:
@@ -88,15 +91,15 @@ def test_change_and_undo(differ):
     # Parse the function and a.
     differ.initialize(func_before + 'a')
     # Parse just b.
-    differ.parse(func_before + 'b', copies=1, parsers=1)
+    differ.parse(func_before + 'b', copies=1, parsers=2)
     # b has changed to a again, so parse that.
-    differ.parse(func_before + 'a', copies=1, parsers=1)
+    differ.parse(func_before + 'a', copies=1, parsers=2)
     # Same as before parsers should not be used. Just a simple copy.
     differ.parse(func_before + 'a', copies=1)
 
     # Now that we have a newline at the end, everything is easier in Python
     # syntax, we can parse once and then get a copy.
-    differ.parse(func_before + 'a\n', copies=1, parsers=1)
+    differ.parse(func_before + 'a\n', copies=1, parsers=2)
     differ.parse(func_before + 'a\n', copies=1)
 
     # Getting rid of an old parser: Still no parsers used.
@@ -135,7 +138,7 @@ def test_if_simple(differ):
     differ.initialize(src + 'a')
     differ.parse(src + else_ + "a", copies=0, parsers=1)
 
-    differ.parse(else_, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(else_, parsers=2, expect_error_leaves=True)
     differ.parse(src + else_, parsers=1)
 
 
@@ -152,7 +155,7 @@ def test_func_with_for_and_comment(differ):
         # COMMENT
         a""")
     differ.initialize(src)
-    differ.parse('a\n' + src, copies=1, parsers=2)
+    differ.parse('a\n' + src, copies=1, parsers=3)
 
 
 def test_one_statement_func(differ):
@@ -236,7 +239,7 @@ def test_backslash(differ):
         def y():
             pass
     """)
-    differ.parse(src, parsers=2)
+    differ.parse(src, parsers=1)
 
     src = dedent(r"""
     def first():
@@ -247,7 +250,7 @@ def test_backslash(differ):
     def second():
         pass
     """)
-    differ.parse(src, parsers=1)
+    differ.parse(src, parsers=2)
 
 
 def test_full_copy(differ):
@@ -261,10 +264,10 @@ def test_wrong_whitespace(differ):
     hello
     '''
     differ.initialize(code)
-    differ.parse(code + 'bar\n    ', parsers=3)
+    differ.parse(code + 'bar\n    ', parsers=2, expect_error_leaves=True)
 
     code += """abc(\npass\n    """
-    differ.parse(code, parsers=2, copies=1, expect_error_leaves=True)
+    differ.parse(code, parsers=2, expect_error_leaves=True)
 
 
 def test_issues_with_error_leaves(differ):
@@ -279,7 +282,7 @@ def test_issues_with_error_leaves(differ):
         str
     ''')
     differ.initialize(code)
-    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code2, parsers=1, expect_error_leaves=True)
 
 
 def test_unfinished_nodes(differ):
@@ -299,7 +302,7 @@ def test_unfinished_nodes(differ):
     a(1)
     ''')
     differ.initialize(code)
-    differ.parse(code2, parsers=1, copies=2)
+    differ.parse(code2, parsers=2, copies=2)
 
 
 def test_nested_if_and_scopes(differ):
@@ -365,7 +368,7 @@ def test_totally_wrong_whitespace(differ):
     '''
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=4, copies=0, expect_error_leaves=True)
+    differ.parse(code2, parsers=2, copies=0, expect_error_leaves=True)
 
 
 def test_node_insertion(differ):
@@ -439,7 +442,7 @@ def test_in_class_movements(differ):
     """)
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=2, copies=1)
+    differ.parse(code2, parsers=1)
 
 
 def test_in_parentheses_newlines(differ):
@@ -484,7 +487,7 @@ def test_indentation_issue(differ):
     """)
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=1)
+    differ.parse(code2, parsers=2)
 
 
 def test_endmarker_newline(differ):
@@ -585,7 +588,7 @@ def test_if_removal_and_reappearence(differ):
             la
     ''')
     differ.initialize(code1)
-    differ.parse(code2, parsers=1, copies=4, expect_error_leaves=True)
+    differ.parse(code2, parsers=3, copies=2, expect_error_leaves=True)
     differ.parse(code1, parsers=1, copies=1)
     differ.parse(code3, parsers=1, copies=1)
 
@@ -618,8 +621,8 @@ def test_differing_docstrings(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=3, copies=1)
-    differ.parse(code1, parsers=3, copies=1)
+    differ.parse(code2, parsers=2, copies=1)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_one_call_in_function_change(differ):
@@ -649,7 +652,7 @@ def test_one_call_in_function_change(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
     differ.parse(code1, parsers=2, copies=1)
 
 
@@ -711,7 +714,7 @@ def test_docstring_removal(differ):
 
     differ.initialize(code1)
     differ.parse(code2, parsers=1, copies=2)
-    differ.parse(code1, parsers=2, copies=1)
+    differ.parse(code1, parsers=3, copies=1)
 
 
 def test_paren_in_strange_position(differ):
@@ -783,7 +786,7 @@ def test_parentheses_before_method(differ):
 
     differ.initialize(code1)
     differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=1, copies=1)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_indentation_issues(differ):
@@ -824,10 +827,10 @@ def test_indentation_issues(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=2, copies=2, expect_error_leaves=True)
-    differ.parse(code1, copies=2)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
+    differ.parse(code1, copies=1, parsers=2)
     differ.parse(code3, parsers=2, copies=1)
-    differ.parse(code1, parsers=1, copies=2)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_error_dedent_issues(differ):
@@ -860,7 +863,7 @@ def test_error_dedent_issues(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=6, copies=2, expect_error_leaves=True)
+    differ.parse(code2, parsers=3, copies=0, expect_error_leaves=True)
     differ.parse(code1, parsers=1, copies=0)
 
 
@@ -892,8 +895,8 @@ Some'random text: yeah
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=1, copies=1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_many_nested_ifs(differ):
@@ -946,7 +949,7 @@ def test_with_and_funcdef_in_call(differ, prefix):
     code2 = insert_line_into_code(code1, 3, 'def y(self, args):\n')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=3, expect_error_leaves=True)
+    differ.parse(code2, parsers=1, expect_error_leaves=True)
     differ.parse(code1, parsers=1)
 
 
@@ -961,14 +964,10 @@ def test_wrong_backslash(differ):
     code2 = insert_line_into_code(code1, 3, '\\.whl$\n')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=2, copies=2, expect_error_leaves=True)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
     differ.parse(code1, parsers=1, copies=1)
 
 
-def test_comment_change(differ):
-    differ.initialize('')
-
-
 def test_random_unicode_characters(differ):
     """
     Those issues were all found with the fuzzer.
@@ -984,9 +983,9 @@ def test_random_unicode_characters(differ):
     differ.parse(s, parsers=1, expect_error_leaves=True)
     differ.parse('')
     differ.parse(s + '\n', parsers=1, expect_error_leaves=True)
-    differ.parse(u'   result = (\r\f\x17\t\x11res)', parsers=2, expect_error_leaves=True)
+    differ.parse(u'   result = (\r\f\x17\t\x11res)', parsers=1, expect_error_leaves=True)
     differ.parse('')
-    differ.parse('   a( # xx\ndef', parsers=2, expect_error_leaves=True)
+    differ.parse('   a( # xx\ndef', parsers=1, expect_error_leaves=True)
 
 
 def test_dedent_end_positions(differ):
@@ -997,7 +996,7 @@ def test_dedent_end_positions(differ):
                 c = {
                      5}
         ''')
-    code2 = dedent('''\
+    code2 = dedent(u'''\
         if 1:
             if ⌟ഒᜈྡྷṭb:
                 2
@@ -1040,7 +1039,7 @@ def test_random_character_insertion(differ):
             # 4
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=3, expect_error_leaves=True)
+    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
     differ.parse(code1, copies=1, parsers=1)
 
 
@@ -1101,8 +1100,8 @@ def test_all_sorts_of_indentation(differ):
                 end
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=4, expect_error_leaves=True)
-    differ.parse(code1, copies=1, parsers=3)
+    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True)
 
     code3 = dedent('''\
             if 1:
@@ -1112,7 +1111,7 @@ def test_all_sorts_of_indentation(differ):
                    d
         \x00
         ''')
-    differ.parse(code3, parsers=2, expect_error_leaves=True)
+    differ.parse(code3, parsers=1, expect_error_leaves=True)
     differ.parse('')
 
 
@@ -1129,7 +1128,7 @@ def test_dont_copy_dedents_in_beginning(differ):
         ''')
     differ.initialize(code1)
     differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=2)
+    differ.parse(code1, parsers=1, copies=1)
 
 
 def test_dont_copy_error_leaves(differ):
@@ -1149,7 +1148,7 @@ def test_dont_copy_error_leaves(differ):
         ''')
     differ.initialize(code1)
     differ.parse(code2, parsers=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=2)
+    differ.parse(code1, parsers=1)
 
 
 def test_error_dedent_in_between(differ):
@@ -1173,7 +1172,7 @@ def test_error_dedent_in_between(differ):
             z
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
+    differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True)
     differ.parse(code1, copies=1, parsers=2)
 
 
@@ -1199,8 +1198,8 @@ def test_some_other_indentation_issues(differ):
         a
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=2, parsers=1, expect_error_leaves=True)
-    differ.parse(code1, copies=2, parsers=2)
+    differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, copies=1, parsers=1)
 
 
 def test_open_bracket_case1(differ):
@@ -1240,8 +1239,8 @@ def test_open_bracket_case2(differ):
                 d
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True)
-    differ.parse(code1, copies=2, parsers=0, expect_error_leaves=True)
+    differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, copies=0, parsers=1, expect_error_leaves=True)
 
 
 def test_some_weird_removals(differ):
@@ -1266,7 +1265,7 @@ def test_some_weird_removals(differ):
         ''')
     differ.initialize(code1)
     differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
-    differ.parse(code3, copies=1, parsers=2, expect_error_leaves=True)
+    differ.parse(code3, copies=1, parsers=3, expect_error_leaves=True)
     differ.parse(code1, copies=1)
 
 
@@ -1285,3 +1284,467 @@ def test_async_copy(differ):
     differ.initialize(code1)
     differ.parse(code2, copies=1, parsers=1)
     differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True)
+
+
+def test_parent_on_decorator(differ):
+    code1 = dedent('''\
+        class AClass:
+            @decorator()
+                def b_test(self):
+                print("Hello")
+                print("world")
+
+            def a_test(self):
+                pass''')
+    code2 = dedent('''\
+        class AClass:
+            @decorator()
+            def b_test(self):
+                print("Hello")
+                print("world")
+
+            def a_test(self):
+                pass''')
+    differ.initialize(code1)
+    module_node = differ.parse(code2, parsers=1)
+    cls = module_node.children[0]
+    cls_suite = cls.children[-1]
+    assert len(cls_suite.children) == 3
+
+
+def test_wrong_indent_in_def(differ):
+    code1 = dedent('''\
+        def x():
+          a
+          b
+        ''')
+
+    code2 = dedent('''\
+        def x():
+         //
+          b
+          c
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=1)
+
+
+def test_backslash_issue(differ):
+    code1 = dedent('''
+        pre = (
+            '')
+        after = 'instead'
+        ''')
+    code2 = dedent('''
+        pre = (
+            '')
+             \\if 
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=1, copies=1)
+
+
+def test_paren_with_indentation(differ):
+    code1 = dedent('''
+        class C:
+            def f(self, fullname, path=None):
+                x
+
+            def load_module(self, fullname):
+                a
+                for prefix in self.search_path:
+                    try:
+                        b
+                    except ImportError:
+                        c
+                else:
+                    raise
+            def x():
+                pass
+        ''')
+    code2 = dedent('''
+        class C:
+            def f(self, fullname, path=None):
+                x
+
+                    (
+                a
+                for prefix in self.search_path:
+                    try:
+                        b
+                    except ImportError:
+                        c
+                else:
+                    raise
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=3, copies=1)
+
+
+def test_error_dedent_in_function(differ):
+    code1 = dedent('''\
+        def x():
+            a
+            b
+            c
+            d
+        ''')
+    code2 = dedent('''\
+        def x():
+            a
+            b
+          c
+            d
+            e
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+
+
+def test_with_formfeed(differ):
+    code1 = dedent('''\
+        @bla
+        async def foo():
+            1
+            yield from []
+            return
+            return ''
+        ''')
+    code2 = dedent('''\
+        @bla
+        async def foo():
+            1
+        \x0cimport 
+            return
+            return ''
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True)
+
+
+def test_repeating_invalid_indent(differ):
+    code1 = dedent('''\
+        def foo():
+            return
+
+        @bla
+            a
+        def foo():
+            a
+            b
+            c
+        ''')
+    code2 = dedent('''\
+        def foo():
+            return
+
+        @bla
+            a
+            b
+            c
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+
+
+def test_another_random_indent(differ):
+    code1 = dedent('''\
+        def foo():
+            a
+        b
+            c
+            return
+        def foo():
+            d
+        ''')
+    code2 = dedent('''\
+        def foo():
+            a
+            c
+            return
+        def foo():
+            d
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=3)
+
+
+def test_invalid_function(differ):
+    code1 = dedent('''\
+        a
+        def foo():
+        def foo():
+            b
+        ''')
+    code2 = dedent('''\
+        a
+        def foo():
+        def foo():
+            b
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_async_func2(differ):
+    code1 = dedent('''\
+        async def foo():
+            return ''
+        @bla
+        async def foo():
+            x
+        ''')
+    code2 = dedent('''\
+        async def foo():
+            return ''
+
+          {
+        @bla
+        async def foo():
+            x
+          y
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True)
+
+
+def test_weird_ending(differ):
+    code1 = dedent('''\
+        def foo():
+            a
+            return
+        ''')
+    code2 = dedent('''\
+        def foo():
+            a
+           nonlocal xF"""
+        y"""''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_nested_class(differ):
+    code1 = dedent('''\
+def c():
+    a = 3
+        class X:
+            b
+        ''')
+    code2 = dedent('''\
+def c():
+    a = 3
+        class X:
+ elif
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_class_with_paren_breaker(differ):
+    code1 = dedent('''\
+class Grammar:
+    x
+    def parse():
+        y
+        parser(
+        )
+        z
+        ''')
+    code2 = dedent('''\
+class Grammar:
+    x
+    def parse():
+        y
+        parser(
+   finally ;
+        )
+        z
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
+
+
+def test_byte_order_mark(differ):
+    code2 = dedent('''\
+
+        x
+        \ufeff
+                 else :
+        ''')
+    differ.initialize('\n')
+    differ.parse(code2, parsers=2, expect_error_leaves=True)
+
+    code3 = dedent('''\
+        \ufeff
+                 if:
+
+        x
+        ''')
+    differ.initialize('\n')
+    differ.parse(code3, parsers=2, expect_error_leaves=True)
+
+
+def test_byte_order_mark2(differ):
+    code = u'\ufeff# foo'
+    differ.initialize(code)
+    differ.parse(code + 'x', parsers=ANY)
+
+
+def test_byte_order_mark3(differ):
+    code1 = u"\ufeff#\ny\n"
+    code2 = u'x\n\ufeff#\n\ufeff#\ny\n'
+    differ.initialize(code1)
+    differ.parse(code2, expect_error_leaves=True, parsers=ANY, copies=ANY)
+    differ.parse(code1, parsers=1)
+
+
+def test_backslash_insertion(differ):
+    code1 = dedent('''
+        def f():
+            x
+            def g():
+                base = "" \\
+                       ""
+                return
+        ''')
+    code2 = dedent('''
+        def f():
+            x
+            def g():
+                base = "" \\
+        def h():
+                       ""
+                return
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=2, copies=1)
+
+
+def test_fstring_with_error_leaf(differ):
+    code1 = dedent("""\
+        def f():
+            x
+        def g():
+            y
+        """)
+    code2 = dedent("""\
+        def f():
+            x
+            F'''
+        def g():
+            y
+            {a
+        \x01
+        """)
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_yet_another_backslash(differ):
+    code1 = dedent('''\
+        def f():
+            x
+            def g():
+                y
+                base = "" \\
+                       "" % to
+                return
+        ''')
+    code2 = dedent('''\
+        def f():
+            x
+            def g():
+                y
+                base = "" \\
+          \x0f
+                return
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True)
+    differ.parse(code1, parsers=ANY, copies=ANY)
+
+
+def test_backslash_before_def(differ):
+    code1 = dedent('''\
+        def f():
+            x
+
+        def g():
+            y
+            z
+        ''')
+    code2 = dedent('''\
+        def f():
+            x
+         >\\
+        def g():
+            y
+         x
+            z
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
+
+
+def test_backslash_with_imports(differ):
+    code1 = dedent('''\
+        from x import y, \\
+        ''')
+    code2 = dedent('''\
+        from x import y, \\
+            z
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1)
+    differ.parse(code1, parsers=1)
+
+
+def test_one_line_function_error_recovery(differ):
+    code1 = dedent('''\
+        class X:
+            x
+            def y(): word """
+                # a
+                # b
+                c(self)
+        ''')
+    code2 = dedent('''\
+        class X:
+            x
+            def y(): word """
+                # a
+                # b
+                c(\x01+self)
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_one_line_property_error_recovery(differ):
+    code1 = dedent('''\
+        class X:
+            x
+            @property
+            def encoding(self): True -
+                return 1
+        ''')
+    code2 = dedent('''\
+        class X:
+            x
+            @property
+            def encoding(self): True -
+                return 1
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)

test/test_error_recovery.py

@@ -1,3 +1,5 @@
+from textwrap import dedent
+
 from parso import parse, load_grammar
 
 
@@ -83,3 +85,65 @@ def test_invalid_token_in_fstr():
     assert error1.type == 'error_leaf'
     assert error2.value == '"'
     assert error2.type == 'error_leaf'
+
+
+def test_dedent_issues1():
+    code = dedent('''\
+        class C:
+            @property
+                f
+                    g
+            end
+        ''')
+    module = load_grammar(version='3.8').parse(code)
+    klass, endmarker = module.children
+    suite = klass.children[-1]
+    assert suite.children[2].type == 'error_leaf'
+    assert suite.children[3].get_code(include_prefix=False) == 'f\n'
+    assert suite.children[5].get_code(include_prefix=False) == 'g\n'
+    assert suite.type == 'suite'
+
+
+def test_dedent_issues2():
+    code = dedent('''\
+        class C:
+            @property
+                if 1:
+                    g
+                else:
+                    h
+            end
+        ''')
+    module = load_grammar(version='3.8').parse(code)
+    klass, endmarker = module.children
+    suite = klass.children[-1]
+    assert suite.children[2].type == 'error_leaf'
+    if_ = suite.children[3]
+    assert if_.children[0] == 'if'
+    assert if_.children[3].type == 'suite'
+    assert if_.children[3].get_code() == '\n            g\n'
+    assert if_.children[4] == 'else'
+    assert if_.children[6].type == 'suite'
+    assert if_.children[6].get_code() == '\n            h\n'
+
+    assert suite.children[4].get_code(include_prefix=False) == 'end\n'
+    assert suite.type == 'suite'
+
+
+def test_dedent_issues3():
+    code = dedent('''\
+        class C:
+          f
+         g
+        ''')
+    module = load_grammar(version='3.8').parse(code)
+    klass, endmarker = module.children
+    suite = klass.children[-1]
+    assert len(suite.children) == 4
+    assert suite.children[1].get_code() == '  f\n'
+    assert suite.children[1].type == 'simple_stmt'
+    assert suite.children[2].get_code() == ''
+    assert suite.children[2].type == 'error_leaf'
+    assert suite.children[2].token_type == 'ERROR_DEDENT'
+    assert suite.children[3].get_code() == ' g\n'
+    assert suite.children[3].type == 'simple_stmt'

test/test_get_code.py

@@ -118,3 +118,16 @@ def test_carriage_return_at_end(code, types):
     assert tree.get_code() == code
     assert [c.type for c in tree.children] == types
     assert tree.end_pos == (len(code) + 1, 0)
+
+
+@pytest.mark.parametrize('code', [
+    ' ',
+    '    F"""',
+    '    F"""\n',
+    '    F""" \n',
+    '    F""" \n3',
+    '    f"""\n"""',
+    '    f"""\n"""\n',
+])
+def test_full_code_round_trip(code):
+    assert parse(code).get_code() == code

test/test_load_grammar.py

@@ -20,7 +20,7 @@ def test_parse_version(string, result):
     assert utils._parse_version(string) == result
 
 
-@pytest.mark.parametrize('string', ['1.', 'a', '#', '1.3.4.5', '1.12'])
+@pytest.mark.parametrize('string', ['1.', 'a', '#', '1.3.4.5'])
 def test_invalid_grammar_version(string):
     with pytest.raises(ValueError):
         load_grammar(version=string)

test/test_parser.py

@@ -194,6 +194,9 @@ def test_no_error_nodes(each_version):
 def test_named_expression(works_ge_py38):
     works_ge_py38.parse("(a := 1, a + 1)")
 
+def test_extended_rhs_annassign(works_ge_py38):
+    works_ge_py38.parse("x: y = z,")
+    works_ge_py38.parse("x: Tuple[int, ...] = z, *q, w")
 
 @pytest.mark.parametrize(
     'param_code', [
@@ -208,3 +211,13 @@ def test_named_expression(works_ge_py38):
 )
 def test_positional_only_arguments(works_ge_py38, param_code):
     works_ge_py38.parse("def x(%s): pass" % param_code)
+
+@pytest.mark.parametrize(
+    'expression', [
+        'a + a',
+        'lambda x: x',
+        'a := lambda x: x'
+    ]
+)
+def test_decorator_expression(works_ge_py39, expression):
+    works_ge_py39.parse("@%s\ndef x(): pass" % expression)

test/test_pgen2.py

@@ -29,13 +29,17 @@ def _invalid_syntax(code, version=None, **kwargs):
         print(module.children)
 
 
-def test_formfeed(each_py2_version):
-    s = u"""print 1\n\x0Cprint 2\n"""
-    t = _parse(s, each_py2_version)
-    assert t.children[0].children[0].type == 'print_stmt'
-    assert t.children[1].children[0].type == 'print_stmt'
-    s = u"""1\n\x0C\x0C2\n"""
-    t = _parse(s, each_py2_version)
+def test_formfeed(each_version):
+    s = u"foo\n\x0c\nfoo\n"
+    t = _parse(s, each_version)
+    assert t.children[0].children[0].type == 'name'
+    assert t.children[1].children[0].type == 'name'
+    s = u"1\n\x0c\x0c\n2\n"
+    t = _parse(s, each_version)
+
+    with pytest.raises(ParserSyntaxError):
+        s = u"\n\x0c2\n"
+        _parse(s, each_version)
 
 
 def test_matrix_multiplication_operator(works_ge_py35):
@@ -83,6 +87,39 @@ def test_async_for(works_ge_py35):
     works_ge_py35.parse("async def foo():\n async for a in b: pass")
 
 
+@pytest.mark.parametrize("body", [
+    """[1 async for a in b
+    ]""",
+    """[1 async
+    for a in b
+    ]""",
+    """[
+    1
+    async for a in b
+    ]""",
+    """[
+    1
+    async for a
+    in b
+    ]""",
+    """[
+    1
+    async
+    for
+    a
+    in
+    b
+    ]""",
+    """  [
+    1 async for a in b
+  ]""",
+])
+def test_async_for_comprehension_newline(works_ge_py36, body):
+    # Issue #139
+    works_ge_py36.parse("""async def foo():
+    {}""".format(body))
+
+
 def test_async_with(works_ge_py35):
     works_ge_py35.parse("async def foo():\n async with a: pass")
 

test/test_python_errors.py

@@ -7,6 +7,8 @@ import warnings
 import pytest
 
 import parso
+
+from textwrap import dedent
 from parso._compatibility import is_pypy
 from .failing_examples import FAILING_EXAMPLES, indent, build_nested
 
@@ -185,12 +187,13 @@ def test_statically_nested_blocks():
 
 
 def test_future_import_first():
-    def is_issue(code, *args):
+    def is_issue(code, *args, **kwargs):
         code = code % args
-        return bool(_get_error_list(code))
+        return bool(_get_error_list(code, **kwargs))
 
     i1 = 'from __future__ import division'
     i2 = 'from __future__ import absolute_import'
+    i3 = 'from __future__ import annotations'
     assert not is_issue(i1)
     assert not is_issue(i1 + ';' + i2)
     assert not is_issue(i1 + '\n' + i2)
@@ -201,6 +204,8 @@ def test_future_import_first():
     assert not is_issue('""\n%s;%s', i1, i2)
     assert not is_issue('"";%s;%s ', i1, i2)
     assert not is_issue('"";%s\n%s ', i1, i2)
+    assert not is_issue(i3, version="3.7")
+    assert is_issue(i3, version="3.6")
     assert is_issue('1;' + i1)
     assert is_issue('1\n' + i1)
     assert is_issue('"";1\n' + i1)
@@ -268,6 +273,9 @@ def test_too_many_levels_of_indentation():
     assert not _get_error_list(build_nested('pass', 49, base=base))
     assert _get_error_list(build_nested('pass', 50, base=base))
 
+def test_paren_kwarg():
+    assert _get_error_list("print((sep)=seperator)", version="3.8")
+    assert not _get_error_list("print((sep)=seperator)", version="3.7")
 
 @pytest.mark.parametrize(
     'code', [
@@ -321,3 +329,88 @@ def test_invalid_fstrings(code, message):
 def test_trailing_comma(code):
     errors = _get_error_list(code)
     assert not errors
+
+def test_continue_in_finally():
+    code = dedent('''\
+        for a in [1]:
+            try:
+                pass
+            finally:
+                continue
+        ''')
+    assert not _get_error_list(code, version="3.8")
+    assert _get_error_list(code, version="3.7")
+
+    
+@pytest.mark.parametrize(
+    'template', [
+        "a, b, {target}, c = d",
+        "a, b, *{target}, c = d",
+        "(a, *{target}), c = d",
+        "for x, {target} in y: pass",
+        "for x, q, {target} in y: pass",
+        "for x, q, *{target} in y: pass",
+        "for (x, *{target}), q in y: pass",
+    ]
+)
+@pytest.mark.parametrize(
+    'target', [
+        "True",
+        "False",
+        "None",
+        "__debug__"
+    ]
+)
+def test_forbidden_name(template, target):
+    assert _get_error_list(template.format(target=target), version="3")
+
+
+def test_repeated_kwarg():
+    # python 3.9+ shows which argument is repeated
+    assert (
+        _get_error_list("f(q=1, q=2)", version="3.8")[0].message
+        == "SyntaxError: keyword argument repeated"
+    )
+    assert (
+        _get_error_list("f(q=1, q=2)", version="3.9")[0].message
+        == "SyntaxError: keyword argument repeated: q"
+    )
+
+
+@pytest.mark.parametrize(
+    ('source', 'no_errors'), [
+        ('a(a for a in b,)', False),
+        ('a(a for a in b, a)', False),
+        ('a(a, a for a in b)', False),
+        ('a(a, b, a for a in b, c, d)', False),
+        ('a(a for a in b)', True),
+        ('a((a for a in b), c)', True),
+        ('a(c, (a for a in b))', True),
+        ('a(a, b, (a for a in b), c, d)', True),
+    ]
+)
+def test_unparenthesized_genexp(source, no_errors):
+    assert bool(_get_error_list(source)) ^ no_errors
+
+@pytest.mark.parametrize(
+    ('source', 'no_errors'), [
+        ('*x = 2', False),
+        ('(*y) = 1', False),
+        ('((*z)) = 1', False),
+        ('a, *b = 1', True),
+        ('a, *b, c = 1', True),
+        ('a, (*b), c = 1', True),
+        ('a, ((*b)), c = 1', True),
+        ('a, (*b, c), d = 1', True),
+        ('[*(1,2,3)]', True),
+        ('{*(1,2,3)}', True),
+        ('[*(1,2,3),]', True),
+        ('[*(1,2,3), *(4,5,6)]', True),
+        ('[0, *(1,2,3)]', True),
+        ('{*(1,2,3),}', True),
+        ('{*(1,2,3), *(4,5,6)}', True),
+        ('{0, *(4,5,6)}', True)
+    ]
+)
+def test_starred_expr(source, no_errors):
+    assert bool(_get_error_list(source, version="3")) ^ no_errors

test/test_tokenize.py

@@ -4,7 +4,6 @@ import sys
 from textwrap import dedent
 
 import pytest
-import sys
 
 from parso.utils import split_lines, parse_version_string
 from parso.python.token import PythonTokenTypes
@@ -239,7 +238,7 @@ xfail_py2 = dict(marks=[pytest.mark.xfail(sys.version_info[0] == 2, reason='Pyth
         (' foo', [INDENT, NAME, DEDENT]),
         ('  foo\n bar', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
         ('  foo\n bar \n baz', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME,
-                                NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
+                                NEWLINE, NAME, DEDENT]),
         (' foo\nbar', [INDENT, NAME, NEWLINE, DEDENT, NAME]),
 
         # Name stuff
@@ -250,6 +249,21 @@ xfail_py2 = dict(marks=[pytest.mark.xfail(sys.version_info[0] == 2, reason='Pyth
         pytest.param(u'²', [ERRORTOKEN], **xfail_py2),
         pytest.param(u'ä²ö', [NAME, ERRORTOKEN, NAME], **xfail_py2),
         pytest.param(u'ää²¹öö', [NAME, ERRORTOKEN, NAME], **xfail_py2),
+        (' \x00a', [INDENT, ERRORTOKEN, NAME, DEDENT]),
+        (dedent('''\
+            class BaseCache:
+                    a
+                def
+                    b
+                def
+                    c
+            '''), [NAME, NAME, OP, NEWLINE, INDENT, NAME, NEWLINE,
+                   ERROR_DEDENT, NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT,
+                   NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, DEDENT]),
+        ('  )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
+        ('a\n b\n  )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP,
+                            NEWLINE, DEDENT, NAME, DEDENT]),
+        (' 1 \\\ndef', [INDENT, NUMBER, NAME, DEDENT]),
     ]
 )
 def test_token_types(code, types):
@@ -258,7 +272,7 @@ def test_token_types(code, types):
 
 
 def test_error_string():
-    t1, newline, endmarker = _get_token_list(' "\n')
+    indent, t1, newline, token, endmarker = _get_token_list(' "\n')
     assert t1.type == ERRORTOKEN
     assert t1.prefix == ' '
     assert t1.string == '"'
@@ -319,16 +333,18 @@ def test_brackets_no_indentation():
 
 
 def test_form_feed():
-    error_token, endmarker = _get_token_list(dedent('''\
+    indent, error_token, dedent_, endmarker = _get_token_list(dedent('''\
         \f"""'''))
     assert error_token.prefix == '\f'
     assert error_token.string == '"""'
     assert endmarker.prefix == ''
+    assert indent.type == INDENT
+    assert dedent_.type == DEDENT
 
 
 def test_carriage_return():
     lst = _get_token_list(' =\\\rclass')
-    assert [t.type for t in lst] == [INDENT, OP, DEDENT, NAME, ENDMARKER]
+    assert [t.type for t in lst] == [INDENT, OP, NAME, DEDENT, ENDMARKER]
 
 
 def test_backslash():
@@ -339,6 +355,7 @@ def test_backslash():
 
 @pytest.mark.parametrize(
     ('code', 'types'), [
+        # f-strings
         ('f"', [FSTRING_START]),
         ('f""', [FSTRING_START, FSTRING_END]),
         ('f" {}"', [FSTRING_START, FSTRING_STRING, OP, OP, FSTRING_END]),
@@ -394,7 +411,7 @@ def test_backslash():
         ]),
     ]
 )
-def test_fstring(code, types, version_ge_py36):
+def test_fstring_token_types(code, types, version_ge_py36):
     actual_types = [t.type for t in _get_token_list(code, version_ge_py36)]
     assert types + [ENDMARKER] == actual_types
 
@@ -414,3 +431,13 @@ def test_fstring(code, types, version_ge_py36):
 def test_fstring_assignment_expression(code, types, version_ge_py38):
     actual_types = [t.type for t in _get_token_list(code, version_ge_py38)]
     assert types + [ENDMARKER] == actual_types
+
+
+def test_fstring_end_error_pos(version_ge_py38):
+    f_start, f_string, bracket, f_end, endmarker = \
+        _get_token_list('f" { "', version_ge_py38)
+    assert f_start.start_pos == (1, 0)
+    assert f_string.start_pos == (1, 2)
+    assert bracket.start_pos == (1, 3)
+    assert f_end.start_pos == (1, 5)
+    assert endmarker.start_pos == (1, 6)

test/test_utils.py

@@ -1,6 +1,11 @@
 from codecs import BOM_UTF8
 
-from parso.utils import split_lines, python_bytes_to_unicode
+from parso.utils import (
+    split_lines,
+    parse_version_string,
+    python_bytes_to_unicode,
+)
+
 import parso
 
 import pytest
@@ -63,3 +68,35 @@ def test_utf8_bom():
     expr_stmt = module.children[0]
     assert expr_stmt.type == 'expr_stmt'
     assert unicode_bom == expr_stmt.get_first_leaf().prefix
+
+
+@pytest.mark.parametrize(
+    ('code', 'errors'), [
+        (b'# coding: wtf-12\nfoo', 'strict'),
+        (b'# coding: wtf-12\nfoo', 'replace'),
+    ]
+)
+def test_bytes_to_unicode_failing_encoding(code, errors):
+    if errors == 'strict':
+        with pytest.raises(LookupError):
+            python_bytes_to_unicode(code, errors=errors)
+    else:
+        python_bytes_to_unicode(code, errors=errors)
+
+@pytest.mark.parametrize(
+    ('version_str', 'version'), [
+        ('3', (3,)),
+        ('3.6', (3, 6)),
+        ('3.6.10', (3, 6)),
+        ('3.10', (3, 10)),
+        ('3.10a9', (3, 10)),
+        ('3.10b9', (3, 10)),
+        ('3.10rc9', (3, 10)),
+    ]
+)
+def test_parse_version_string(version_str, version):
+    parsed_version = parse_version_string(version_str)
+    if len(version) == 1:
+        assert parsed_version[0] == version[0]
+    else:
+        assert parsed_version == version