Fix a failing test

Let consumers know that we have type annotations

.coveragerc

@@ -1,4 +1,5 @@
 [run]
+source = parso
 
 [report]
 # Regexes for lines to exclude from consideration

.gitignore

@@ -11,3 +11,4 @@ parso.egg-info/
 /.cache/
 /.pytest_cache
 test/fuzz-redo.pickle
+/venv/

.travis.yml

@@ -1,25 +1,28 @@
+dist: xenial
 language: python
-sudo: false
 python:
-  - 2.6
   - 2.7
   - 3.4
   - 3.5
   - 3.6
-  - pypy
+  - 3.7
+  - 3.8.2
+  - nightly
+  - pypy2.7-6.0
+  - pypy3.5-6.0
 matrix:
-  include:
-    - { python: "3.7", dist: xenial, sudo: true }
-    - python: 3.5
-      env: TOXENV=cov
   allow_failures:
-    - env: TOXENV=cov
+    - python: nightly
+  include:
+    - python: 3.5
+      env: TOXENV=py35-coverage
 install:
     - pip install --quiet tox-travis
 script:
     - tox
 after_script:
-    - if [ $TOXENV == "cov" ]; then
-      pip install --quiet coveralls;
-      coveralls;
+    - |
+      if [ "${TOXENV%-coverage}" == "$TOXENV" ]; then
+        pip install --quiet coveralls;
+        coveralls;
       fi

AUTHORS.txt

@@ -49,6 +49,9 @@ Mathias Rav (@Mortal) <rav@cs.au.dk>
 Daniel Fiterman (@dfit99) <fitermandaniel2@gmail.com>
 Simon Ruggier (@sruggier)
 Élie Gouzien (@ElieGouzien)
+Tim Gates (@timgates42) <tim.gates@iress.com>
+Batuhan Taskaya (@isidentical) <isidentical@gmail.com>
+Jocelyn Boullier (@Kazy) <jocelyn@boullier.bzh>
 
 
 Note: (@user) means a github user name.

CHANGELOG.rst

@@ -3,7 +3,83 @@
 Changelog
 ---------
 
-0.3.2 (2018-01-24)
+0.7.1 (2020-07-24)
+++++++++++++++++++
+
+- Fixed a couple of smaller bugs (mostly syntax error detection in
+  ``Grammar.iter_errors``)
+
+This is going to be the last release that supports Python 2.7, 3.4 and 3.5.
+
+0.7.0 (2020-04-13)
+++++++++++++++++++
+
+- Fix a lot of annoying bugs in the diff parser. The fuzzer did not find
+  issues anymore even after running it for more than 24 hours (500k tests).
+- Small grammar change: suites can now contain newlines even after a newline.
+  This should really not matter if you don't use error recovery. It allows for
+  nicer error recovery.
+
+0.6.2 (2020-02-27)
+++++++++++++++++++
+
+- Bugfixes
+- Add Grammar.refactor (might still be subject to change until 0.7.0)
+
+0.6.1 (2020-02-03)
+++++++++++++++++++
+
+- Add ``parso.normalizer.Issue.end_pos`` to make it possible to know where an
+  issue ends
+
+0.6.0 (2020-01-26)
+++++++++++++++++++
+
+- Dropped Python 2.6/Python 3.3 support
+- del_stmt names are now considered as a definition
+  (for ``name.is_definition()``)
+- Bugfixes
+
+0.5.2 (2019-12-15)
+++++++++++++++++++
+
+- Add include_setitem to get_definition/is_definition and get_defined_names (#66)
+- Fix named expression error listing (#89, #90)
+- Fix some f-string tokenizer issues (#93)
+
+0.5.1 (2019-07-13)
+++++++++++++++++++
+
+- Fix: Some unicode identifiers were not correctly tokenized
+- Fix: Line continuations in f-strings are now working
+
+0.5.0 (2019-06-20)
+++++++++++++++++++
+
+- **Breaking Change** comp_for is now called sync_comp_for for all Python
+  versions to be compatible with the Python 3.8 Grammar
+- Added .pyi stubs for a lot of the parso API
+- Small FileIO changes
+
+0.4.0 (2019-04-05)
+++++++++++++++++++
+
+- Python 3.8 support
+- FileIO support, it's now possible to use abstract file IO, support is alpha
+
+0.3.4 (2019-02-13)
++++++++++++++++++++
+
+- Fix an f-string tokenizer error
+
+0.3.3 (2019-02-06)
++++++++++++++++++++
+
+- Fix async errors in the diff parser
+- A fix in iter_errors
+- This is a very small bugfix release
+
+0.3.2 (2019-01-24)
 +++++++++++++++++++
 
 - 20+ bugfixes in the diff parser and 3 in the tokenizer

README.rst

@@ -11,6 +11,10 @@ parso - A Python Parser
     :target: https://coveralls.io/github/davidhalter/parso?branch=master
     :alt: Coverage Status
 
+.. image:: https://pepy.tech/badge/parso
+    :target: https://pepy.tech/project/parso
+    :alt: PyPI Downloads
+
 .. image:: https://raw.githubusercontent.com/davidhalter/parso/master/docs/_static/logo_characters.png
 
 Parso is a Python parser that supports error recovery and round-trip parsing

conftest.py

@@ -13,8 +13,8 @@ from parso.utils import parse_version_string
 
 collect_ignore = ["setup.py"]
 
-VERSIONS_2 = '2.6', '2.7'
-VERSIONS_3 = '3.3', '3.4', '3.5', '3.6', '3.7'
+VERSIONS_2 = '2.7',
+VERSIONS_3 = '3.4', '3.5', '3.6', '3.7', '3.8'
 
 
 @pytest.fixture(scope='session')
@@ -58,7 +58,9 @@ def pytest_generate_tests(metafunc):
     elif 'each_py3_version' in metafunc.fixturenames:
         metafunc.parametrize('each_py3_version', VERSIONS_3)
     elif 'version_ge_py36' in metafunc.fixturenames:
-        metafunc.parametrize('version_ge_py36', ['3.6', '3.7'])
+        metafunc.parametrize('version_ge_py36', ['3.6', '3.7', '3.8'])
+    elif 'version_ge_py38' in metafunc.fixturenames:
+        metafunc.parametrize('version_ge_py38', ['3.8'])
 
 
 class NormalizerIssueCase(object):
@@ -85,12 +87,12 @@ def pytest_configure(config):
         root = logging.getLogger()
         root.setLevel(logging.DEBUG)
 
-        ch = logging.StreamHandler(sys.stdout)
-        ch.setLevel(logging.DEBUG)
+        #ch = logging.StreamHandler(sys.stdout)
+        #ch.setLevel(logging.DEBUG)
         #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
         #ch.setFormatter(formatter)
 
-        root.addHandler(ch)
+        #root.addHandler(ch)
 
 
 class Checker():
@@ -155,3 +157,18 @@ def works_ge_py3(each_version):
 def works_ge_py35(each_version):
     version_info = parse_version_string(each_version)
     return Checker(each_version, version_info >= (3, 5))
+
+@pytest.fixture
+def works_ge_py36(each_version):
+    version_info = parse_version_string(each_version)
+    return Checker(each_version, version_info >= (3, 6))
+
+@pytest.fixture
+def works_ge_py38(each_version):
+    version_info = parse_version_string(each_version)
+    return Checker(each_version, version_info >= (3, 8))
+
+@pytest.fixture
+def works_ge_py39(each_version):
+    version_info = parse_version_string(each_version)
+    return Checker(each_version, version_info >= (3, 9))

deploy-master.sh

@@ -26,7 +26,7 @@ git checkout $BRANCH
 tox
 
 # Create tag
-tag=v$(python -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
+tag=v$(python3 -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
 
 master_ref=$(git show-ref -s heads/$BRANCH)
 tag_ref=$(git show-ref -s $tag || true)
@@ -43,7 +43,7 @@ fi
 # Package and upload to PyPI
 #rm -rf dist/ - Not needed anymore, because the folder is never reused.
 echo `pwd`
-python setup.py sdist bdist_wheel
+python3 setup.py sdist bdist_wheel
 # Maybe do a pip install twine before.
 twine upload dist/*
 

parso/__init__.py

@@ -43,7 +43,7 @@ from parso.grammar import Grammar, load_grammar
 from parso.utils import split_lines, python_bytes_to_unicode
 
 
-__version__ = '0.3.2'
+__version__ = '0.7.1'
 
 
 def parse(code=None, **kwargs):

parso/__init__.pyi

@@ -0,0 +1,19 @@
+from typing import Any, Optional, Union
+
+from parso.grammar import Grammar as Grammar, load_grammar as load_grammar
+from parso.parser import ParserSyntaxError as ParserSyntaxError
+from parso.utils import python_bytes_to_unicode as python_bytes_to_unicode, split_lines as split_lines
+
+__version__: str = ...
+
+def parse(
+    code: Optional[Union[str, bytes]],
+    *,
+    version: Optional[str] = None,
+    error_recovery: bool = True,
+    path: Optional[str] = None,
+    start_symbol: Optional[str] = None,
+    cache: bool = False,
+    diff_cache: bool = False,
+    cache_path: Optional[str] = None,
+) -> Any: ...

parso/_compatibility.py

@@ -1,14 +1,11 @@
 """
-To ensure compatibility from Python ``2.6`` - ``3.3``, a module has been
+To ensure compatibility from Python ``2.7`` - ``3.3``, a module has been
 created. Clearly there is huge need to use conforming syntax.
 """
+import os
 import sys
 import platform
 
-# Cannot use sys.version.major and minor names, because in Python 2.6 it's not
-# a namedtuple.
-py_version = int(str(sys.version_info[0]) + str(sys.version_info[1]))
-
 # unicode function
 try:
     unicode = unicode
@@ -39,7 +36,7 @@ def u(string):
     have to cast back to a unicode (and we know that we always deal with valid
     unicode, because we check that in the beginning).
     """
-    if py_version >= 30:
+    if sys.version_info.major >= 3:
         return str(string)
 
     if not isinstance(string, unicode):
@@ -48,9 +45,17 @@ def u(string):
 
 
 try:
+    # Python 3.3+
     FileNotFoundError = FileNotFoundError
 except NameError:
-    FileNotFoundError = IOError
+    # Python 2.7 (both IOError + OSError)
+    FileNotFoundError = EnvironmentError
+try:
+    # Python 3.3+
+    PermissionError = PermissionError
+except NameError:
+    # Python 2.7 (both IOError + OSError)
+    PermissionError = EnvironmentError
 
 
 def utf8_repr(func):
@@ -65,39 +70,32 @@ def utf8_repr(func):
         else:
             return result
 
-    if py_version >= 30:
+    if sys.version_info.major >= 3:
         return func
     else:
         return wrapper
 
 
-try:
-    from functools import total_ordering
-except ImportError:
-    # Python 2.6
-    def total_ordering(cls):
-        """Class decorator that fills in missing ordering methods"""
-        convert = {
-            '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
-                       ('__le__', lambda self, other: self < other or self == other),
-                       ('__ge__', lambda self, other: not self < other)],
-            '__le__': [('__ge__', lambda self, other: not self <= other or self == other),
-                       ('__lt__', lambda self, other: self <= other and not self == other),
-                       ('__gt__', lambda self, other: not self <= other)],
-            '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
-                       ('__ge__', lambda self, other: self > other or self == other),
-                       ('__le__', lambda self, other: not self > other)],
-            '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
-                       ('__gt__', lambda self, other: self >= other and not self == other),
-                       ('__lt__', lambda self, other: not self >= other)]
-        }
-        roots = set(dir(cls)) & set(convert)
-        if not roots:
-            raise ValueError('must define at least one ordering operation: < > <= >=')
-        root = max(roots)       # prefer __lt__ to __le__ to __gt__ to __ge__
-        for opname, opfunc in convert[root]:
-            if opname not in roots:
-                opfunc.__name__ = opname
-                opfunc.__doc__ = getattr(int, opname).__doc__
-                setattr(cls, opname, opfunc)
-        return cls
+if sys.version_info < (3, 5):
+    """
+    A super-minimal shim around listdir that behave like
+    scandir for the information we need.
+    """
+    class _DirEntry:
+
+        def __init__(self, name, basepath):
+            self.name = name
+            self.basepath = basepath
+
+        @property
+        def path(self):
+            return os.path.join(self.basepath, self.name)
+
+        def stat(self):
+            # won't follow symlinks
+            return os.lstat(os.path.join(self.basepath, self.name))
+
+    def scandir(dir):
+        return [_DirEntry(name, dir) for name in os.listdir(dir)]
+else:
+    from os import scandir

parso/cache.py

@@ -7,18 +7,42 @@ import shutil
 import platform
 import errno
 import logging
+import warnings
 
 try:
     import cPickle as pickle
 except:
     import pickle
 
-from parso._compatibility import FileNotFoundError
+from parso._compatibility import FileNotFoundError, PermissionError, scandir
+from parso.file_io import FileIO
 
 LOG = logging.getLogger(__name__)
 
+_CACHED_FILE_MINIMUM_SURVIVAL = 60 * 10  # 10 minutes
+"""
+Cached files should survive at least a few minutes.
+"""
 
-_PICKLE_VERSION = 30
+_CACHED_FILE_MAXIMUM_SURVIVAL = 60 * 60 * 24 * 30
+"""
+Maximum time for a cached file to survive if it is not
+accessed within.
+"""
+
+_CACHED_SIZE_TRIGGER = 600
+"""
+This setting limits the amount of cached files. It's basically a way to start
+garbage collection.
+
+The reasoning for this limit being as big as it is, is the following:
+
+Numpy, Pandas, Matplotlib and Tensorflow together use about 500 files. This
+makes Jedi use ~500mb of memory. Since we might want a bit more than those few
+libraries, we just increase it a bit.
+"""
+
+_PICKLE_VERSION = 33
 """
 Version number (integer) for file system cache.
 
@@ -40,20 +64,23 @@ _VERSION_TAG = '%s-%s%s-%s' % (
 """
 Short name for distinguish Python implementations and versions.
 
-It's like `sys.implementation.cache_tag` but for Python < 3.3
+It's like `sys.implementation.cache_tag` but for Python2
 we generate something similar.  See:
 http://docs.python.org/3/library/sys.html#sys.implementation
 """
 
+
 def _get_default_cache_path():
     if platform.system().lower() == 'windows':
-        dir_ = os.path.join(os.getenv('LOCALAPPDATA') or '~', 'Parso', 'Parso')
+        dir_ = os.path.join(os.getenv('LOCALAPPDATA')
+                            or os.path.expanduser('~'), 'Parso', 'Parso')
     elif platform.system().lower() == 'darwin':
         dir_ = os.path.join('~', 'Library', 'Caches', 'Parso')
     else:
         dir_ = os.path.join(os.getenv('XDG_CACHE_HOME') or '~/.cache', 'parso')
     return os.path.expanduser(dir_)
 
+
 _default_cache_path = _get_default_cache_path()
 """
 The path where the cache is stored.
@@ -64,6 +91,19 @@ On Linux, if environment variable ``$XDG_CACHE_HOME`` is set,
 ``$XDG_CACHE_HOME/parso`` is used instead of the default one.
 """
 
+_CACHE_CLEAR_THRESHOLD = 60 * 60 * 24
+
+def _get_cache_clear_lock(cache_path = None):
+    """
+    The path where the cache lock is stored.
+
+    Cache lock will prevent continous cache clearing and only allow garbage
+    collection once a day (can be configured in _CACHE_CLEAR_THRESHOLD).
+    """
+    cache_path = cache_path or _get_default_cache_path()
+    return FileIO(os.path.join(cache_path, "PARSO-CACHE-LOCK"))
+
+
 parser_cache = {}
 
 
@@ -74,23 +114,29 @@ class _NodeCacheItem(object):
         if change_time is None:
             change_time = time.time()
         self.change_time = change_time
+        self.last_used = change_time
 
 
-def load_module(hashed_grammar, path, cache_path=None):
+def load_module(hashed_grammar, file_io, cache_path=None):
     """
     Returns a module or None, if it fails.
     """
-    try:
-        p_time = os.path.getmtime(path)
-    except FileNotFoundError:
+    p_time = file_io.get_last_modified()
+    if p_time is None:
         return None
 
     try:
-        module_cache_item = parser_cache[hashed_grammar][path]
+        module_cache_item = parser_cache[hashed_grammar][file_io.path]
         if p_time <= module_cache_item.change_time:
+            module_cache_item.last_used = time.time()
             return module_cache_item.node
     except KeyError:
-        return _load_from_file_system(hashed_grammar, path, p_time, cache_path=cache_path)
+        return _load_from_file_system(
+            hashed_grammar,
+            file_io.path,
+            p_time,
+            cache_path=cache_path
+        )
 
 
 def _load_from_file_system(hashed_grammar, path, p_time, cache_path=None):
@@ -116,22 +162,50 @@ def _load_from_file_system(hashed_grammar, path, p_time, cache_path=None):
     except FileNotFoundError:
         return None
     else:
-        parser_cache.setdefault(hashed_grammar, {})[path] = module_cache_item
+        _set_cache_item(hashed_grammar, path, module_cache_item)
         LOG.debug('pickle loaded: %s', path)
         return module_cache_item.node
 
 
-def save_module(hashed_grammar, path, module, lines, pickling=True, cache_path=None):
+def _set_cache_item(hashed_grammar, path, module_cache_item):
+    if sum(len(v) for v in parser_cache.values()) >= _CACHED_SIZE_TRIGGER:
+        # Garbage collection of old cache files.
+        # We are basically throwing everything away that hasn't been accessed
+        # in 10 minutes.
+        cutoff_time = time.time() - _CACHED_FILE_MINIMUM_SURVIVAL
+        for key, path_to_item_map in parser_cache.items():
+            parser_cache[key] = {
+                path: node_item
+                for path, node_item in path_to_item_map.items()
+                if node_item.last_used > cutoff_time
+            }
+
+    parser_cache.setdefault(hashed_grammar, {})[path] = module_cache_item
+
+
+def try_to_save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None):
+    path = file_io.path
     try:
-        p_time = None if path is None else os.path.getmtime(path)
+        p_time = None if path is None else file_io.get_last_modified()
     except OSError:
         p_time = None
         pickling = False
 
     item = _NodeCacheItem(module, lines, p_time)
-    parser_cache.setdefault(hashed_grammar, {})[path] = item
+    _set_cache_item(hashed_grammar, path, item)
     if pickling and path is not None:
-        _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path)
+        try:
+            _save_to_file_system(hashed_grammar, path, item, cache_path=cache_path)
+        except PermissionError:
+            # It's not really a big issue if the cache cannot be saved to the
+            # file system. It's still in RAM in that case. However we should
+            # still warn the user that this is happening.
+            warnings.warn(
+                'Tried to save a file to %s, but got permission denied.',
+                Warning
+            )
+        else:
+            _remove_cache_and_update_lock(cache_path=cache_path)
 
 
 def _save_to_file_system(hashed_grammar, path, item, cache_path=None):
@@ -146,6 +220,46 @@ def clear_cache(cache_path=None):
     parser_cache.clear()
 
 
+def clear_inactive_cache(
+    cache_path=None,
+    inactivity_threshold=_CACHED_FILE_MAXIMUM_SURVIVAL,
+):
+    if cache_path is None:
+        cache_path = _get_default_cache_path()
+    if not os.path.exists(cache_path):
+        return False
+    for version_path in os.listdir(cache_path):
+        version_path = os.path.join(cache_path, version_path)
+        if not os.path.isdir(version_path):
+            continue
+        for file in scandir(version_path):
+            if (
+                file.stat().st_atime + _CACHED_FILE_MAXIMUM_SURVIVAL
+                <= time.time()
+            ):
+                try:
+                    os.remove(file.path)
+                except OSError: # silently ignore all failures
+                    continue
+    else:
+        return True
+
+
+def _remove_cache_and_update_lock(cache_path = None):
+    lock = _get_cache_clear_lock(cache_path=cache_path)
+    clear_lock_time = lock.get_last_modified()
+    if (
+        clear_lock_time is None # first time
+        or clear_lock_time + _CACHE_CLEAR_THRESHOLD <= time.time()
+    ):
+        if not lock._touch():
+            # First make sure that as few as possible other cleanup jobs also
+            # get started. There is still a race condition but it's probably
+            # not a big problem.
+            return False
+
+        clear_inactive_cache(cache_path = cache_path)
+
 def _get_hashed_path(hashed_grammar, path, cache_path=None):
     directory = _get_cache_directory_path(cache_path=cache_path)
 

parso/file_io.py

@@ -0,0 +1,47 @@
+import os
+from parso._compatibility import FileNotFoundError
+
+
+class FileIO(object):
+    def __init__(self, path):
+        self.path = path
+
+    def read(self):  # Returns bytes/str
+        # We would like to read unicode here, but we cannot, because we are not
+        # sure if it is a valid unicode file. Therefore just read whatever is
+        # here.
+        with open(self.path, 'rb') as f:
+            return f.read()
+
+    def get_last_modified(self):
+        """
+        Returns float - timestamp or None, if path doesn't exist.
+        """
+        try:
+            return os.path.getmtime(self.path)
+        except OSError:
+            # Might raise FileNotFoundError, OSError for Python 2
+            return None
+
+    def _touch(self):
+        try:
+            os.utime(self.path, None)
+        except FileNotFoundError:
+            try:
+                file = open(self.path, 'a')
+                file.close()
+            except (OSError, IOError):  # TODO Maybe log this?
+                return False
+        return True
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self.path)
+
+
+class KnownContentFileIO(FileIO):
+    def __init__(self, path, content):
+        super(KnownContentFileIO, self).__init__(path)
+        self._content = content
+
+    def read(self):
+        return self._content

parso/grammar.py

@@ -7,11 +7,13 @@ from parso.utils import split_lines, python_bytes_to_unicode, parse_version_stri
 from parso.python.diff import DiffParser
 from parso.python.tokenize import tokenize_lines, tokenize
 from parso.python.token import PythonTokenTypes
-from parso.cache import parser_cache, load_module, save_module
+from parso.cache import parser_cache, load_module, try_to_save_module
 from parso.parser import BaseParser
 from parso.python.parser import Parser as PythonParser
 from parso.python.errors import ErrorFinderConfig
 from parso.python import pep8
+from parso.file_io import FileIO, KnownContentFileIO
+from parso.normalizer import RefactoringNormalizer
 
 _loaded_grammars = {}
 
@@ -56,7 +58,8 @@ class Grammar(object):
         :param str path: The path to the file you want to open. Only needed for caching.
         :param bool cache: Keeps a copy of the parser tree in RAM and on disk
             if a path is given. Returns the cached trees if the corresponding
-            files on disk have not changed.
+            files on disk have not changed. Note that this stores pickle files
+            on your file system (e.g. for Linux in ``~/.cache/parso/``).
         :param bool diff_cache: Diffs the cached python module against the new
             code and tries to parse only the parts that have changed. Returns
             the same (changed) module that is found in cache. Using this option
@@ -77,14 +80,14 @@ class Grammar(object):
 
     def _parse(self, code=None, error_recovery=True, path=None,
                start_symbol=None, cache=False, diff_cache=False,
-               cache_path=None, start_pos=(1, 0)):
+               cache_path=None, file_io=None, start_pos=(1, 0)):
         """
         Wanted python3.5 * operator and keyword only arguments. Therefore just
         wrap it all.
         start_pos here is just a parameter internally used. Might be public
         sometime in the future.
         """
-        if code is None and path is None:
+        if code is None and path is None and file_io is None:
             raise TypeError("Please provide either code or a path.")
 
         if start_symbol is None:
@@ -93,15 +96,19 @@ class Grammar(object):
         if error_recovery and start_symbol != 'file_input':
             raise NotImplementedError("This is currently not implemented.")
 
-        if cache and path is not None:
-            module_node = load_module(self._hashed, path, cache_path=cache_path)
+        if file_io is None:
+            if code is None:
+                file_io = FileIO(path)
+            else:
+                file_io = KnownContentFileIO(path, code)
+
+        if cache and file_io.path is not None:
+            module_node = load_module(self._hashed, file_io, cache_path=cache_path)
             if module_node is not None:
                 return module_node
 
         if code is None:
-            with open(path, 'rb') as f:
-                code = f.read()
-
+            code = file_io.read()
         code = python_bytes_to_unicode(code)
 
         lines = split_lines(code, keepends=True)
@@ -110,7 +117,7 @@ class Grammar(object):
                 raise TypeError("You have to define a diff parser to be able "
                                 "to use this option.")
             try:
-                module_cache_item = parser_cache[self._hashed][path]
+                module_cache_item = parser_cache[self._hashed][file_io.path]
             except KeyError:
                 pass
             else:
@@ -125,13 +132,13 @@ class Grammar(object):
                     old_lines=old_lines,
                     new_lines=lines
                 )
-                save_module(self._hashed, path, new_node, lines,
+                try_to_save_module(self._hashed, file_io, new_node, lines,
                             # Never pickle in pypy, it's slow as hell.
                             pickling=cache and not is_pypy,
                             cache_path=cache_path)
                 return new_node
 
-        tokens = self._tokenizer(lines, start_pos)
+        tokens = self._tokenizer(lines, start_pos=start_pos)
 
         p = self._parser(
             self._pgen_grammar,
@@ -141,7 +148,7 @@ class Grammar(object):
         root_node = p.parse(tokens=tokens)
 
         if cache or diff_cache:
-            save_module(self._hashed, path, root_node, lines,
+            try_to_save_module(self._hashed, file_io, root_node, lines,
                         # Never pickle in pypy, it's slow as hell.
                         pickling=cache and not is_pypy,
                         cache_path=cache_path)
@@ -164,6 +171,9 @@ class Grammar(object):
 
         return self._get_normalizer_issues(node, self._error_normalizer_config)
 
+    def refactor(self, base_node, node_to_str_map):
+        return RefactoringNormalizer(node_to_str_map).walk(base_node)
+
     def _get_normalizer(self, normalizer_config):
         if normalizer_config is None:
             normalizer_config = self._default_normalizer_config
@@ -205,8 +215,8 @@ class PythonGrammar(Grammar):
         )
         self.version_info = version_info
 
-    def _tokenize_lines(self, lines, start_pos):
-        return tokenize_lines(lines, self.version_info, start_pos=start_pos)
+    def _tokenize_lines(self, lines, **kwargs):
+        return tokenize_lines(lines, self.version_info, **kwargs)
 
     def _tokenize(self, code):
         # Used by Jedi.
@@ -218,7 +228,7 @@ def load_grammar(**kwargs):
     Loads a :py:class:`parso.Grammar`. The default version is the current Python
     version.
 
-    :param str version: A python version string, e.g. ``version='3.3'``.
+    :param str version: A python version string, e.g. ``version='3.8'``.
     :param str path: A path to a grammar file
     """
     def load_grammar(language='python', version=None, path=None):
@@ -242,7 +252,7 @@ def load_grammar(**kwargs):
                     grammar = PythonGrammar(version_info, bnf_text)
                     return _loaded_grammars.setdefault(path, grammar)
                 except FileNotFoundError:
-                    message = "Python version %s is currently not supported." % version
+                    message = "Python version %s.%s is currently not supported." % (version_info.major, version_info.minor)
                     raise NotImplementedError(message)
         else:
             raise NotImplementedError("No support for language %s." % language)

parso/grammar.pyi

@@ -0,0 +1,38 @@
+from typing import Any, Callable, Generic, Optional, Sequence, TypeVar, Union
+from typing_extensions import Literal
+
+from parso.utils import PythonVersionInfo
+
+_Token = Any
+_NodeT = TypeVar("_NodeT")
+
+class Grammar(Generic[_NodeT]):
+    _default_normalizer_config: Optional[Any] = ...
+    _error_normalizer_config: Optional[Any] = None
+    _start_nonterminal: str = ...
+    _token_namespace: Optional[str] = None
+    def __init__(
+        self,
+        text: str,
+        tokenizer: Callable[[Sequence[str], int], Sequence[_Token]],
+        parser: Any = ...,
+        diff_parser: Any = ...,
+    ) -> None: ...
+    def parse(
+        self,
+        code: Union[str, bytes] = ...,
+        error_recovery: bool = ...,
+        path: Optional[str] = ...,
+        start_symbol: Optional[str] = ...,
+        cache: bool = ...,
+        diff_cache: bool = ...,
+        cache_path: Optional[str] = ...,
+    ) -> _NodeT: ...
+
+class PythonGrammar(Grammar):
+    version_info: PythonVersionInfo
+    def __init__(self, version_info: PythonVersionInfo, bnf_text: str) -> None: ...
+
+def load_grammar(
+    language: Literal["python"] = "python", version: Optional[str] = ..., path: str = ...
+) -> Grammar: ...

parso/normalizer.py

@@ -12,6 +12,9 @@ class _NormalizerMeta(type):
 
 
 class Normalizer(use_metaclass(_NormalizerMeta)):
+    _rule_type_instances = {}
+    _rule_value_instances = {}
+
     def __init__(self, grammar, config):
         self.grammar = grammar
         self._config = config
@@ -41,8 +44,8 @@ class Normalizer(use_metaclass(_NormalizerMeta)):
         except AttributeError:
             return self.visit_leaf(node)
         else:
-           with self.visit_node(node):
-               return ''.join(self.visit(child) for child in children)
+            with self.visit_node(node):
+                return ''.join(self.visit(child) for child in children)
 
     @contextmanager
     def visit_node(self, node):
@@ -119,7 +122,6 @@ class NormalizerConfig(object):
 
 class Issue(object):
     def __init__(self, node, code, message):
-        self._node = node
         self.code = code
         """
         An integer code that stands for the type of error.
@@ -133,6 +135,7 @@ class Issue(object):
         The start position position of the error as a tuple (line, column). As
         always in |parso| the first line is 1 and the first column 0.
         """
+        self.end_pos = node.end_pos
 
     def __eq__(self, other):
         return self.start_pos == other.start_pos and self.code == other.code
@@ -147,7 +150,6 @@ class Issue(object):
         return '<%s: %s>' % (self.__class__.__name__, self.code)
 
 
-
 class Rule(object):
     code = None
     message = None
@@ -161,7 +163,7 @@ class Rule(object):
     def get_node(self, node):
         return node
 
-    def _get_message(self, message):
+    def _get_message(self, message, node):
         if message is None:
             message = self.message
             if message is None:
@@ -174,7 +176,7 @@ class Rule(object):
             if code is None:
                 raise ValueError("The error code on the class is not set.")
 
-        message = self._get_message(message)
+        message = self._get_message(message, node)
 
         self._normalizer.add_issue(node, code, message)
 
@@ -182,3 +184,20 @@ class Rule(object):
         if self.is_issue(node):
             issue_node = self.get_node(node)
             self.add_issue(issue_node)
+
+
+class RefactoringNormalizer(Normalizer):
+    def __init__(self, node_to_str_map):
+        self._node_to_str_map = node_to_str_map
+
+    def visit(self, node):
+        try:
+            return self._node_to_str_map[node]
+        except KeyError:
+            return super(RefactoringNormalizer, self).visit(node)
+
+    def visit_leaf(self, leaf):
+        try:
+            return self._node_to_str_map[leaf]
+        except KeyError:
+            return super(RefactoringNormalizer, self).visit_leaf(leaf)

parso/parser.py

@@ -134,7 +134,7 @@ class BaseParser(object):
                 # However, the error recovery might have added the token again, if
                 # the stack is empty, we're fine.
                 raise InternalParseError(
-                    "incomplete input", token.type, token.value, token.start_pos
+                    "incomplete input", token.type, token.string, token.start_pos
                 )
 
             if len(self.stack) > 1:

parso/pgen2/__init__.pyi

@@ -0,0 +1 @@
+from parso.pgen2.generator import generate_grammar as generate_grammar

parso/pgen2/generator.py

@@ -212,7 +212,8 @@ def _dump_nfa(start, finish):
     todo = [start]
     for i, state in enumerate(todo):
         print("  State", i, state is finish and "(final)" or "")
-        for label, next_ in state.arcs:
+        for arc in state.arcs:
+            label, next_ = arc.nonterminal_or_string, arc.next
             if next_ in todo:
                 j = todo.index(next_)
             else:
@@ -244,7 +245,7 @@ def generate_grammar(bnf_grammar, token_namespace):
     rule_to_dfas = {}
     start_nonterminal = None
     for nfa_a, nfa_z in GrammarParser(bnf_grammar).parse():
-        #_dump_nfa(a, z)
+        #_dump_nfa(nfa_a, nfa_z)
         dfas = _make_dfas(nfa_a, nfa_z)
         #_dump_dfas(dfas)
         # oldlen = len(dfas)
@@ -309,13 +310,39 @@ def _calculate_tree_traversal(nonterminal_to_dfas):
             _calculate_first_plans(nonterminal_to_dfas, first_plans, nonterminal)
 
     # Now that we have calculated the first terminals, we are sure that
-    # there is no left recursion or ambiguities.
+    # there is no left recursion.
 
     for dfas in nonterminal_to_dfas.values():
         for dfa_state in dfas:
+            transitions = dfa_state.transitions
             for nonterminal, next_dfa in dfa_state.nonterminal_arcs.items():
                 for transition, pushes in first_plans[nonterminal].items():
-                    dfa_state.transitions[transition] = DFAPlan(next_dfa, pushes)
+                    if transition in transitions:
+                        prev_plan = transitions[transition]
+                        # Make sure these are sorted so that error messages are
+                        # at least deterministic
+                        choices = sorted([
+                            (
+                                prev_plan.dfa_pushes[0].from_rule
+                                if prev_plan.dfa_pushes
+                                else prev_plan.next_dfa.from_rule
+                            ),
+                            (
+                                pushes[0].from_rule
+                                if pushes else next_dfa.from_rule
+                            ),
+                        ])
+                        raise ValueError(
+                            "Rule %s is ambiguous; given a %s token, we "
+                            "can't determine if we should evaluate %s or %s."
+                            % (
+                                (
+                                    dfa_state.from_rule,
+                                    transition,
+                                ) + tuple(choices)
+                            )
+                        )
+                    transitions[transition] = DFAPlan(next_dfa, pushes)
 
 
 def _calculate_first_plans(nonterminal_to_dfas, first_plans, nonterminal):
@@ -345,13 +372,6 @@ def _calculate_first_plans(nonterminal_to_dfas, first_plans, nonterminal):
                 raise ValueError("left recursion for rule %r" % nonterminal)
 
         for t, pushes in first_plans2.items():
-            check = new_first_plans.get(t)
-            if check is not None:
-                raise ValueError(
-                    "Rule %s is ambiguous; %s is the"
-                    " start of the rule %s as well as %s."
-                    % (nonterminal, t, nonterminal2, check[-1].from_rule)
-                )
             new_first_plans[t] = [next_] + pushes
 
     first_plans[nonterminal] = new_first_plans

parso/pgen2/generator.pyi

@@ -0,0 +1,38 @@
+from typing import Any, Generic, Mapping, Sequence, Set, TypeVar, Union
+
+from parso.pgen2.grammar_parser import NFAState
+
+_TokenTypeT = TypeVar("_TokenTypeT")
+
+class Grammar(Generic[_TokenTypeT]):
+    nonterminal_to_dfas: Mapping[str, Sequence[DFAState[_TokenTypeT]]]
+    reserved_syntax_strings: Mapping[str, ReservedString]
+    start_nonterminal: str
+    def __init__(
+        self,
+        start_nonterminal: str,
+        rule_to_dfas: Mapping[str, Sequence[DFAState]],
+        reserved_syntax_strings: Mapping[str, ReservedString],
+    ) -> None: ...
+
+class DFAPlan:
+    next_dfa: DFAState
+    dfa_pushes: Sequence[DFAState]
+
+class DFAState(Generic[_TokenTypeT]):
+    from_rule: str
+    nfa_set: Set[NFAState]
+    is_final: bool
+    arcs: Mapping[str, DFAState]  # map from all terminals/nonterminals to DFAState
+    nonterminal_arcs: Mapping[str, DFAState]
+    transitions: Mapping[Union[_TokenTypeT, ReservedString], DFAPlan]
+    def __init__(
+        self, from_rule: str, nfa_set: Set[NFAState], final: NFAState
+    ) -> None: ...
+
+class ReservedString:
+    value: str
+    def __init__(self, value: str) -> None: ...
+    def __repr__(self) -> str: ...
+
+def generate_grammar(bnf_grammar: str, token_namespace: Any) -> Grammar[Any]: ...

parso/pgen2/grammar_parser.py

@@ -141,6 +141,9 @@ class NFAArc(object):
         self.next = next_
         self.nonterminal_or_string = nonterminal_or_string
 
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.nonterminal_or_string)
+
 
 class NFAState(object):
     def __init__(self, from_rule):

parso/pgen2/grammar_parser.pyi

@@ -0,0 +1,20 @@
+from typing import Generator, List, Optional, Tuple
+
+from parso.python.token import TokenType
+
+class GrammarParser:
+    generator: Generator[TokenType, None, None]
+    def __init__(self, bnf_grammar: str) -> None: ...
+    def parse(self) -> Generator[Tuple[NFAState, NFAState], None, None]: ...
+
+class NFAArc:
+    next: NFAState
+    nonterminal_or_string: Optional[str]
+    def __init__(
+        self, next_: NFAState, nonterminal_or_string: Optional[str]
+    ) -> None: ...
+
+class NFAState:
+    from_rule: str
+    arcs: List[NFAArc]
+    def __init__(self, from_rule: str) -> None: ...

parso/python/diff.py

@@ -1,9 +1,29 @@
 """
-Basically a contains parser that is faster, because it tries to parse only
-parts and if anything changes, it only reparses the changed parts.
+The diff parser is trying to be a faster version of the normal parser by trying
+to reuse the nodes of a previous pass over the same file. This is also called
+incremental parsing in parser literature. The difference is mostly that with
+incremental parsing you get a range that needs to be reparsed. Here we
+calculate that range ourselves by using difflib. After that it's essentially
+incremental parsing.
 
-It works with a simple diff in the beginning and will try to reuse old parser
-fragments.
+The biggest issue of this approach is that we reuse nodes in a mutable way. The
+intial design and idea is quite problematic for this parser, but it is also
+pretty fast. Measurements showed that just copying nodes in Python is simply
+quite a bit slower (especially for big files >3 kLOC). Therefore we did not
+want to get rid of the mutable nodes, since this is usually not an issue.
+
+This is by far the hardest software I ever wrote, exactly because the initial
+design is crappy. When you have to account for a lot of mutable state, it
+creates a ton of issues that you would otherwise not have. This file took
+probably 3-6 months to write, which is insane for a parser.
+
+There is a fuzzer in that helps test this whole thing. Please use it if you
+make changes here. If you run the fuzzer like::
+
+    test/fuzz_diff_parser.py random -n 100000
+
+you can be pretty sure that everything is still fine. I sometimes run the
+fuzzer up to 24h to make sure everything is still ok.
 """
 import re
 import difflib
@@ -13,7 +33,7 @@ import logging
 from parso.utils import split_lines
 from parso.python.parser import Parser
 from parso.python.tree import EndMarker
-from parso.python.tokenize import PythonToken
+from parso.python.tokenize import PythonToken, BOM_UTF8_STRING
 from parso.python.token import PythonTokenTypes
 
 LOG = logging.getLogger(__name__)
@@ -21,21 +41,37 @@ DEBUG_DIFF_PARSER = False
 
 _INDENTATION_TOKENS = 'INDENT', 'ERROR_DEDENT', 'DEDENT'
 
+NEWLINE = PythonTokenTypes.NEWLINE
+DEDENT = PythonTokenTypes.DEDENT
+NAME = PythonTokenTypes.NAME
+ERROR_DEDENT = PythonTokenTypes.ERROR_DEDENT
+ENDMARKER = PythonTokenTypes.ENDMARKER
+
+
+def _is_indentation_error_leaf(node):
+    return node.type == 'error_leaf' and node.token_type in _INDENTATION_TOKENS
+
 
 def _get_previous_leaf_if_indentation(leaf):
-    while leaf and leaf.type == 'error_leaf' \
-            and leaf.token_type in _INDENTATION_TOKENS:
+    while leaf and _is_indentation_error_leaf(leaf):
         leaf = leaf.get_previous_leaf()
     return leaf
 
 
 def _get_next_leaf_if_indentation(leaf):
-    while leaf and leaf.type == 'error_leaf' \
-            and leaf.token_type in _INDENTATION_TOKENS:
-        leaf = leaf.get_previous_leaf()
+    while leaf and _is_indentation_error_leaf(leaf):
+        leaf = leaf.get_next_leaf()
     return leaf
 
 
+def _get_suite_indentation(tree_node):
+    return _get_indentation(tree_node.children[1])
+
+
+def _get_indentation(tree_node):
+    return tree_node.start_pos[1]
+
+
 def _assert_valid_graph(node):
     """
     Checks if the parent/children relationship is correct.
@@ -70,6 +106,10 @@ def _assert_valid_graph(node):
             actual = line, len(splitted[-1])
         else:
             actual = previous_start_pos[0], previous_start_pos[1] + len(content)
+            if content.startswith(BOM_UTF8_STRING) \
+                    and node.get_start_pos_of_prefix() == (1, 0):
+                # Remove the byte order mark
+                actual = actual[0], actual[1] - 1
 
         assert node.start_pos == actual, (node.start_pos, actual)
     else:
@@ -78,6 +118,26 @@ def _assert_valid_graph(node):
             _assert_valid_graph(child)
 
 
+def _assert_nodes_are_equal(node1, node2):
+    try:
+        children1 = node1.children
+    except AttributeError:
+        assert not hasattr(node2, 'children'), (node1, node2)
+        assert node1.value == node2.value, (node1, node2)
+        assert node1.type == node2.type, (node1, node2)
+        assert node1.prefix == node2.prefix, (node1, node2)
+        assert node1.start_pos == node2.start_pos, (node1, node2)
+        return
+    else:
+        try:
+            children2 = node2.children
+        except AttributeError:
+            assert False, (node1, node2)
+    for n1, n2 in zip(children1, children2):
+        _assert_nodes_are_equal(n1, n2)
+    assert len(children1) == len(children2), '\n' + repr(children1) + '\n' + repr(children2)
+
+
 def _get_debug_error_message(module, old_lines, new_lines):
     current_lines = split_lines(module.get_code(), keepends=True)
     current_diff = difflib.unified_diff(new_lines, current_lines)
@@ -95,6 +155,15 @@ def _get_last_line(node_or_leaf):
     if _ends_with_newline(last_leaf):
         return last_leaf.start_pos[0]
     else:
+        n = last_leaf.get_next_leaf()
+        if n.type == 'endmarker' and '\n' in n.prefix:
+            # This is a very special case and has to do with error recovery in
+            # Parso. The problem is basically that there's no newline leaf at
+            # the end sometimes (it's required in the grammar, but not needed
+            # actually before endmarker, CPython just adds a newline to make
+            # source code pass the parser, to account for that Parso error
+            # recovery allows small_stmt instead of simple_stmt).
+            return last_leaf.end_pos[0] + 1
         return last_leaf.end_pos[0]
 
 
@@ -233,7 +302,7 @@ class DiffParser(object):
 
             if operation == 'equal':
                 line_offset = j1 - i1
-                self._copy_from_old_parser(line_offset, i2, j2)
+                self._copy_from_old_parser(line_offset, i1 + 1, i2, j2)
             elif operation == 'replace':
                 self._parse(until_line=j2)
             elif operation == 'insert':
@@ -249,8 +318,14 @@ class DiffParser(object):
             # If there is reasonable suspicion that the diff parser is not
             # behaving well, this should be enabled.
             try:
-                assert self._module.get_code() == ''.join(new_lines)
+                code = ''.join(new_lines)
+                assert self._module.get_code() == code
                 _assert_valid_graph(self._module)
+                without_diff_parser_module = Parser(
+                    self._pgen_grammar,
+                    error_recovery=True
+                ).parse(self._tokenizer(new_lines))
+                _assert_nodes_are_equal(self._module, without_diff_parser_module)
             except AssertionError:
                 print(_get_debug_error_message(self._module, old_lines, new_lines))
                 raise
@@ -268,7 +343,7 @@ class DiffParser(object):
         if self._module.get_code() != ''.join(lines_new):
             LOG.warning('parser issue:\n%s\n%s', ''.join(old_lines), ''.join(lines_new))
 
-    def _copy_from_old_parser(self, line_offset, until_line_old, until_line_new):
+    def _copy_from_old_parser(self, line_offset, start_line_old, until_line_old, until_line_new):
         last_until_line = -1
         while until_line_new > self._nodes_tree.parsed_until_line:
             parsed_until_line_old = self._nodes_tree.parsed_until_line - line_offset
@@ -282,12 +357,18 @@ class DiffParser(object):
                 p_children = line_stmt.parent.children
                 index = p_children.index(line_stmt)
 
-                from_ = self._nodes_tree.parsed_until_line + 1
-                copied_nodes = self._nodes_tree.copy_nodes(
-                    p_children[index:],
-                    until_line_old,
-                    line_offset
-                )
+                if start_line_old == 1 \
+                        and p_children[0].get_first_leaf().prefix.startswith(BOM_UTF8_STRING):
+                    # If there's a BOM in the beginning, just reparse. It's too
+                    # complicated to account for it otherwise.
+                    copied_nodes = []
+                else:
+                    from_ = self._nodes_tree.parsed_until_line + 1
+                    copied_nodes = self._nodes_tree.copy_nodes(
+                        p_children[index:],
+                        until_line_old,
+                        line_offset
+                    )
                 # Match all the nodes that are in the wanted range.
                 if copied_nodes:
                     self._copy_count += 1
@@ -333,7 +414,10 @@ class DiffParser(object):
             node = self._try_parse_part(until_line)
             nodes = node.children
 
-            self._nodes_tree.add_parsed_nodes(nodes)
+            self._nodes_tree.add_parsed_nodes(nodes, self._keyword_token_indents)
+            if self._replace_tos_indent is not None:
+                self._nodes_tree.indents[-1] = self._replace_tos_indent
+
             LOG.debug(
                 'parse_part from %s to %s (to %s in part parser)',
                 nodes[0].get_start_pos_of_prefix()[0],
@@ -369,34 +453,39 @@ class DiffParser(object):
         return self._active_parser.parse(tokens=tokens)
 
     def _diff_tokenize(self, lines, until_line, line_offset=0):
-        is_first_token = True
-        omitted_first_indent = False
-        indents = []
-        tokens = self._tokenizer(lines, (1, 0))
-        stack = self._active_parser.stack
-        for typ, string, start_pos, prefix in tokens:
-            start_pos = start_pos[0] + line_offset, start_pos[1]
-            if typ == PythonTokenTypes.INDENT:
-                indents.append(start_pos[1])
-                if is_first_token:
-                    omitted_first_indent = True
-                    # We want to get rid of indents that are only here because
-                    # we only parse part of the file. These indents would only
-                    # get parsed as error leafs, which doesn't make any sense.
-                    is_first_token = False
-                    continue
-            is_first_token = False
+        was_newline = False
+        indents = self._nodes_tree.indents
+        initial_indentation_count = len(indents)
 
-            # In case of omitted_first_indent, it might not be dedented fully.
-            # However this is a sign for us that a dedent happened.
-            if typ == PythonTokenTypes.DEDENT \
-                    or typ == PythonTokenTypes.ERROR_DEDENT \
-                    and omitted_first_indent and len(indents) == 1:
-                indents.pop()
-                if omitted_first_indent and not indents:
+        tokens = self._tokenizer(
+            lines,
+            start_pos=(line_offset + 1, 0),
+            indents=indents,
+            is_first_token=line_offset == 0,
+        )
+        stack = self._active_parser.stack
+        self._replace_tos_indent = None
+        self._keyword_token_indents = {}
+        # print('start', line_offset + 1, indents)
+        for token in tokens:
+            # print(token, indents)
+            typ = token.type
+            if typ == DEDENT:
+                if len(indents) < initial_indentation_count:
                     # We are done here, only thing that can come now is an
                     # endmarker or another dedented code block.
-                    typ, string, start_pos, prefix = next(tokens)
+                    while True:
+                        typ, string, start_pos, prefix = token = next(tokens)
+                        if typ in (DEDENT, ERROR_DEDENT):
+                            if typ == ERROR_DEDENT:
+                                # We want to force an error dedent in the next
+                                # parser/pass. To make this possible we just
+                                # increase the location by one.
+                                self._replace_tos_indent = start_pos[1] + 1
+                                pass
+                        else:
+                            break
+
                     if '\n' in prefix or '\r' in prefix:
                         prefix = re.sub(r'[^\n\r]+\Z', '', prefix)
                     else:
@@ -404,36 +493,38 @@ class DiffParser(object):
                         if start_pos[1] - len(prefix) == 0:
                             prefix = ''
                     yield PythonToken(
-                        PythonTokenTypes.ENDMARKER, '',
-                        (start_pos[0] + line_offset, 0),
+                        ENDMARKER, '',
+                        start_pos,
                         prefix
                     )
                     break
-            elif typ == PythonTokenTypes.NEWLINE and start_pos[0] >= until_line:
-                yield PythonToken(typ, string, start_pos, prefix)
-                # Check if the parser is actually in a valid suite state.
-                if _suite_or_file_input_is_valid(self._pgen_grammar, stack):
-                    start_pos = start_pos[0] + 1, 0
-                    while len(indents) > int(omitted_first_indent):
-                        indents.pop()
-                        yield PythonToken(PythonTokenTypes.DEDENT, '', start_pos, '')
+            elif typ == NEWLINE and token.start_pos[0] >= until_line:
+                was_newline = True
+            elif was_newline:
+                was_newline = False
+                if len(indents) == initial_indentation_count:
+                    # Check if the parser is actually in a valid suite state.
+                    if _suite_or_file_input_is_valid(self._pgen_grammar, stack):
+                        yield PythonToken(ENDMARKER, '', token.start_pos, '')
+                        break
 
-                    yield PythonToken(PythonTokenTypes.ENDMARKER, '', start_pos, '')
-                    break
-                else:
-                    continue
+            if typ == NAME and token.string in ('class', 'def'):
+                self._keyword_token_indents[token.start_pos] = list(indents)
 
-            yield PythonToken(typ, string, start_pos, prefix)
+            yield token
 
 
 class _NodesTreeNode(object):
-    _ChildrenGroup = namedtuple('_ChildrenGroup', 'prefix children line_offset last_line_offset_leaf')
+    _ChildrenGroup = namedtuple(
+        '_ChildrenGroup',
+        'prefix children line_offset last_line_offset_leaf')
 
-    def __init__(self, tree_node, parent=None):
+    def __init__(self, tree_node, parent=None, indentation=0):
         self.tree_node = tree_node
         self._children_groups = []
         self.parent = parent
         self._node_children = []
+        self.indentation = indentation
 
     def finish(self):
         children = []
@@ -461,10 +552,13 @@ class _NodesTreeNode(object):
     def add_child_node(self, child_node):
         self._node_children.append(child_node)
 
-    def add_tree_nodes(self, prefix, children, line_offset=0, last_line_offset_leaf=None):
+    def add_tree_nodes(self, prefix, children, line_offset=0,
+                       last_line_offset_leaf=None):
         if last_line_offset_leaf is None:
             last_line_offset_leaf = children[-1].get_last_leaf()
-        group = self._ChildrenGroup(prefix, children, line_offset, last_line_offset_leaf)
+        group = self._ChildrenGroup(
+            prefix, children, line_offset, last_line_offset_leaf
+        )
         self._children_groups.append(group)
 
     def get_last_line(self, suffix):
@@ -491,6 +585,9 @@ class _NodesTreeNode(object):
             return max(line, self._node_children[-1].get_last_line(suffix))
         return line
 
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.tree_node)
+
 
 class _NodesTree(object):
     def __init__(self, module):
@@ -499,34 +596,19 @@ class _NodesTree(object):
         self._module = module
         self._prefix_remainder = ''
         self.prefix = ''
+        self.indents = [0]
 
     @property
     def parsed_until_line(self):
         return self._working_stack[-1].get_last_line(self.prefix)
 
-    def _get_insertion_node(self, indentation_node):
-        indentation = indentation_node.start_pos[1]
-
-        # find insertion node
-        while True:
-            node = self._working_stack[-1]
-            tree_node = node.tree_node
-            if tree_node.type == 'suite':
-                # A suite starts with NEWLINE, ...
-                node_indentation = tree_node.children[1].start_pos[1]
-
-                if indentation >= node_indentation:  # Not a Dedent
-                    # We might be at the most outer layer: modules. We
-                    # don't want to depend on the first statement
-                    # having the right indentation.
-                    return node
-
-            elif tree_node.type == 'file_input':
+    def _update_insertion_node(self, indentation):
+        for node in reversed(list(self._working_stack)):
+            if node.indentation < indentation or node is self._working_stack[0]:
                 return node
-
             self._working_stack.pop()
 
-    def add_parsed_nodes(self, tree_nodes):
+    def add_parsed_nodes(self, tree_nodes, keyword_token_indents):
         old_prefix = self.prefix
         tree_nodes = self._remove_endmarker(tree_nodes)
         if not tree_nodes:
@@ -535,23 +617,27 @@ class _NodesTree(object):
 
         assert tree_nodes[0].type != 'newline'
 
-        node = self._get_insertion_node(tree_nodes[0])
+        node = self._update_insertion_node(tree_nodes[0].start_pos[1])
         assert node.tree_node.type in ('suite', 'file_input')
         node.add_tree_nodes(old_prefix, tree_nodes)
         # tos = Top of stack
-        self._update_tos(tree_nodes[-1])
+        self._update_parsed_node_tos(tree_nodes[-1], keyword_token_indents)
 
-    def _update_tos(self, tree_node):
-        if tree_node.type in ('suite', 'file_input'):
-            new_tos = _NodesTreeNode(tree_node)
+    def _update_parsed_node_tos(self, tree_node, keyword_token_indents):
+        if tree_node.type == 'suite':
+            def_leaf = tree_node.parent.children[0]
+            new_tos = _NodesTreeNode(
+                tree_node,
+                indentation=keyword_token_indents[def_leaf.start_pos][-1],
+            )
             new_tos.add_tree_nodes('', list(tree_node.children))
 
             self._working_stack[-1].add_child_node(new_tos)
             self._working_stack.append(new_tos)
 
-            self._update_tos(tree_node.children[-1])
+            self._update_parsed_node_tos(tree_node.children[-1], keyword_token_indents)
         elif _func_or_class_has_suite(tree_node):
-            self._update_tos(tree_node.children[-1])
+            self._update_parsed_node_tos(tree_node.children[-1], keyword_token_indents)
 
     def _remove_endmarker(self, tree_nodes):
         """
@@ -561,7 +647,8 @@ class _NodesTree(object):
         is_endmarker = last_leaf.type == 'endmarker'
         self._prefix_remainder = ''
         if is_endmarker:
-            separation = max(last_leaf.prefix.rfind('\n'), last_leaf.prefix.rfind('\r'))
+            prefix = last_leaf.prefix
+            separation = max(prefix.rfind('\n'), prefix.rfind('\r'))
             if separation > -1:
                 # Remove the whitespace part of the prefix after a newline.
                 # That is not relevant if parentheses were opened. Always parse
@@ -577,6 +664,26 @@ class _NodesTree(object):
             tree_nodes = tree_nodes[:-1]
         return tree_nodes
 
+    def _get_matching_indent_nodes(self, tree_nodes, is_new_suite):
+        # There might be a random dedent where we have to stop copying.
+        # Invalid indents are ok, because the parser handled that
+        # properly before. An invalid dedent can happen, because a few
+        # lines above there was an invalid indent.
+        node_iterator = iter(tree_nodes)
+        if is_new_suite:
+            yield next(node_iterator)
+
+        first_node = next(node_iterator)
+        indent = _get_indentation(first_node)
+        if not is_new_suite and indent not in self.indents:
+            return
+        yield first_node
+
+        for n in node_iterator:
+            if _get_indentation(n) != indent:
+                return
+            yield n
+
     def copy_nodes(self, tree_nodes, until_line, line_offset):
         """
         Copies tree nodes from the old parser tree.
@@ -588,19 +695,38 @@ class _NodesTree(object):
             # issues.
             return []
 
-        self._get_insertion_node(tree_nodes[0])
+        indentation = _get_indentation(tree_nodes[0])
+        old_working_stack = list(self._working_stack)
+        old_prefix = self.prefix
+        old_indents = self.indents
+        self.indents = [i for i in self.indents if i <= indentation]
 
-        new_nodes, self._working_stack, self.prefix = self._copy_nodes(
+        self._update_insertion_node(indentation)
+
+        new_nodes, self._working_stack, self.prefix, added_indents = self._copy_nodes(
             list(self._working_stack),
             tree_nodes,
             until_line,
             line_offset,
             self.prefix,
         )
+        if new_nodes:
+            self.indents += added_indents
+        else:
+            self._working_stack = old_working_stack
+            self.prefix = old_prefix
+            self.indents = old_indents
         return new_nodes
 
-    def _copy_nodes(self, working_stack, nodes, until_line, line_offset, prefix=''):
+    def _copy_nodes(self, working_stack, nodes, until_line, line_offset,
+                    prefix='', is_nested=False):
         new_nodes = []
+        added_indents = []
+
+        nodes = list(self._get_matching_indent_nodes(
+            nodes,
+            is_new_suite=is_nested,
+        ))
 
         new_prefix = ''
         for node in nodes:
@@ -620,26 +746,83 @@ class _NodesTree(object):
                 if _func_or_class_has_suite(node):
                     new_nodes.append(node)
                 break
+            try:
+                c = node.children
+            except AttributeError:
+                pass
+            else:
+                # This case basically appears with error recovery of one line
+                # suites like `def foo(): bar.-`. In this case we might not
+                # include a newline in the statement and we need to take care
+                # of that.
+                n = node
+                if n.type == 'decorated':
+                    n = n.children[-1]
+                if n.type in ('async_funcdef', 'async_stmt'):
+                    n = n.children[-1]
+                if n.type in ('classdef', 'funcdef'):
+                    suite_node = n.children[-1]
+                else:
+                    suite_node = c[-1]
+
+                if suite_node.type in ('error_leaf', 'error_node'):
+                    break
 
             new_nodes.append(node)
 
+        # Pop error nodes at the end from the list
+        if new_nodes:
+            while new_nodes:
+                last_node = new_nodes[-1]
+                if (last_node.type in ('error_leaf', 'error_node')
+                        or _is_flow_node(new_nodes[-1])):
+                    # Error leafs/nodes don't have a defined start/end. Error
+                    # nodes might not end with a newline (e.g. if there's an
+                    # open `(`). Therefore ignore all of them unless they are
+                    # succeeded with valid parser state.
+                    # If we copy flows at the end, they might be continued
+                    # after the copy limit (in the new parser).
+                    # In this while loop we try to remove until we find a newline.
+                    new_prefix = ''
+                    new_nodes.pop()
+                    while new_nodes:
+                        last_node = new_nodes[-1]
+                        if last_node.get_last_leaf().type == 'newline':
+                            break
+                        new_nodes.pop()
+                    continue
+                if len(new_nodes) > 1 and new_nodes[-2].type == 'error_node':
+                    # The problem here is that Parso error recovery sometimes
+                    # influences nodes before this node.
+                    # Since the new last node is an error node this will get
+                    # cleaned up in the next while iteration.
+                    new_nodes.pop()
+                    continue
+                break
+
         if not new_nodes:
-            return [], working_stack, prefix
+            return [], working_stack, prefix, added_indents
 
         tos = working_stack[-1]
         last_node = new_nodes[-1]
         had_valid_suite_last = False
+        # Pop incomplete suites from the list
         if _func_or_class_has_suite(last_node):
             suite = last_node
             while suite.type != 'suite':
                 suite = suite.children[-1]
 
-            suite_tos = _NodesTreeNode(suite)
+            indent = _get_suite_indentation(suite)
+            added_indents.append(indent)
+
+            suite_tos = _NodesTreeNode(suite, indentation=_get_indentation(last_node))
             # Don't need to pass line_offset here, it's already done by the
             # parent.
-            suite_nodes, new_working_stack, new_prefix = self._copy_nodes(
-                working_stack + [suite_tos], suite.children, until_line, line_offset
+            suite_nodes, new_working_stack, new_prefix, ai = self._copy_nodes(
+                working_stack + [suite_tos], suite.children, until_line, line_offset,
+                is_nested=True,
             )
+            added_indents += ai
             if len(suite_nodes) < 2:
                 # A suite only with newline is not valid.
                 new_nodes.pop()
@@ -650,25 +833,6 @@ class _NodesTree(object):
                 working_stack = new_working_stack
                 had_valid_suite_last = True
 
-        if new_nodes:
-            last_node = new_nodes[-1]
-            if (last_node.type in ('error_leaf', 'error_node') or
-                    _is_flow_node(new_nodes[-1])):
-                # Error leafs/nodes don't have a defined start/end. Error
-                # nodes might not end with a newline (e.g. if there's an
-                # open `(`). Therefore ignore all of them unless they are
-                # succeeded with valid parser state.
-                # If we copy flows at the end, they might be continued
-                # after the copy limit (in the new parser).
-                # In this while loop we try to remove until we find a newline.
-                new_prefix = ''
-                new_nodes.pop()
-                while new_nodes:
-                    last_node = new_nodes[-1]
-                    if last_node.get_last_leaf().type == 'newline':
-                        break
-                    new_nodes.pop()
-
         if new_nodes:
             if not _ends_with_newline(new_nodes[-1].get_last_leaf()) and not had_valid_suite_last:
                 p = new_nodes[-1].get_next_leaf().prefix
@@ -682,15 +846,19 @@ class _NodesTree(object):
                 last = new_nodes[-1]
                 if last.type == 'decorated':
                     last = last.children[-1]
+                if last.type in ('async_funcdef', 'async_stmt'):
+                    last = last.children[-1]
                 last_line_offset_leaf = last.children[-2].get_last_leaf()
                 assert last_line_offset_leaf == ':'
             else:
                 last_line_offset_leaf = new_nodes[-1].get_last_leaf()
-            tos.add_tree_nodes(prefix, new_nodes, line_offset, last_line_offset_leaf)
+            tos.add_tree_nodes(
+                prefix, new_nodes, line_offset, last_line_offset_leaf,
+            )
             prefix = new_prefix
             self._prefix_remainder = ''
 
-        return new_nodes, working_stack, prefix
+        return new_nodes, working_stack, prefix, added_indents
 
     def close(self):
         self._base_node.finish()
@@ -706,6 +874,8 @@ class _NodesTree(object):
         lines = split_lines(self.prefix)
         assert len(lines) > 0
         if len(lines) == 1:
+            if lines[0].startswith(BOM_UTF8_STRING) and end_pos == [1, 0]:
+                end_pos[1] -= 1
             end_pos[1] += len(lines[0])
         else:
             end_pos[0] += len(lines) - 1

parso/python/errors.py

@@ -6,7 +6,7 @@ from contextlib import contextmanager
 
 from parso.normalizer import Normalizer, NormalizerConfig, Issue, Rule
 from parso.python.tree import search_ancestor
-from parso.parser import ParserSyntaxError
+from parso.python.tokenize import _get_token_collection
 
 _BLOCK_STMTS = ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt')
 _STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist')
@@ -14,10 +14,84 @@ _STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist')
 _MAX_BLOCK_SIZE = 20
 _MAX_INDENT_COUNT = 100
 ALLOWED_FUTURES = (
-    'all_feature_names', 'nested_scopes', 'generators', 'division',
-    'absolute_import', 'with_statement', 'print_function', 'unicode_literals',
+    'nested_scopes', 'generators', 'division', 'absolute_import',
+    'with_statement', 'print_function', 'unicode_literals',
 )
+_COMP_FOR_TYPES = ('comp_for', 'sync_comp_for')
 
+def _get_rhs_name(node, version):
+    type_ = node.type
+    if type_ == "lambdef":
+        return "lambda"
+    elif type_ == "atom":
+        comprehension = _get_comprehension_type(node)
+        first, second = node.children[:2]
+        if comprehension is not None:
+            return comprehension
+        elif second.type == "dictorsetmaker":
+            if version < (3, 8):
+                return "literal"
+            else:
+                if second.children[1] == ":" or second.children[0] == "**":
+                    return "dict display"
+                else:
+                    return "set display"
+        elif (
+            first == "("
+            and (second == ")"
+            or (len(node.children) == 3 and node.children[1].type == "testlist_comp"))
+        ):
+            return "tuple"
+        elif first == "(":
+            return _get_rhs_name(_remove_parens(node), version=version)
+        elif first == "[":
+            return "list"
+        elif first == "{" and second == "}":
+            return "dict display"
+        elif first == "{" and len(node.children) > 2:
+            return "set display"
+    elif type_ == "keyword":
+        if "yield" in node.value:
+            return "yield expression"
+        if version < (3, 8):
+            return "keyword"
+        else:
+            return str(node.value)
+    elif type_ == "operator" and node.value == "...":
+        return "Ellipsis"
+    elif type_ == "comparison":
+        return "comparison"
+    elif type_ in ("string", "number", "strings"):
+        return "literal"
+    elif type_ == "yield_expr":
+        return "yield expression"
+    elif type_ == "test":
+        return "conditional expression"
+    elif type_ in ("atom_expr", "power"):
+        if node.children[0] == "await":
+            return "await expression"
+        elif node.children[-1].type == "trailer":
+            trailer = node.children[-1]
+            if trailer.children[0] == "(":
+                return "function call"
+            elif trailer.children[0] == "[":
+                return "subscript"
+            elif trailer.children[0] == ".":
+                return "attribute"
+    elif (
+        ("expr" in type_
+        and "star_expr" not in type_)  # is a substring
+        or "_test" in type_
+        or type_ in ("term", "factor")
+    ):
+        return "operator"
+    elif type_ == "star_expr":
+        return "starred"
+    elif type_ == "testlist_star_expr":
+        return "tuple"
+    elif type_ == "fstring":
+        return "f-string expression"
+    return type_ # shouldn't reach here
 
 def _iter_stmts(scope):
     """
@@ -35,12 +109,12 @@ def _iter_stmts(scope):
 
 def _get_comprehension_type(atom):
     first, second = atom.children[:2]
-    if second.type == 'testlist_comp' and second.children[1].type == 'comp_for':
+    if second.type == 'testlist_comp' and second.children[1].type in _COMP_FOR_TYPES:
         if first == '[':
             return 'list comprehension'
         else:
             return 'generator expression'
-    elif second.type == 'dictorsetmaker' and second.children[-1].type == 'comp_for':
+    elif second.type == 'dictorsetmaker' and second.children[-1].type in _COMP_FOR_TYPES:
         if second.children[1] == ':':
             return 'dict comprehension'
         else:
@@ -52,7 +126,7 @@ def _is_future_import(import_from):
     # It looks like a __future__ import that is relative is still a future
     # import. That feels kind of odd, but whatever.
     # if import_from.level != 0:
-        # return False
+    #     return False
     from_names = import_from.get_from_names()
     return [n.value for n in from_names] == ['__future__']
 
@@ -94,19 +168,33 @@ def _is_future_import_first(import_from):
 
 
 def _iter_definition_exprs_from_lists(exprlist):
-    for child in exprlist.children[::2]:
-        if child.type == 'atom' and child.children[0] in ('(', '['):
-            testlist_comp = child.children[0]
-            if testlist_comp.type == 'testlist_comp':
-                for expr in _iter_definition_exprs_from_lists(testlist_comp):
-                    yield expr
-                continue
+    def check_expr(child):
+        if child.type == 'atom':
+            if child.children[0] == '(':
+                testlist_comp = child.children[1]
+                if testlist_comp.type == 'testlist_comp':
+                    for expr in _iter_definition_exprs_from_lists(testlist_comp):
+                        yield expr
+                    return
+                else:
+                    # It's a paren that doesn't do anything, like 1 + (1)
+                    for c in check_expr(testlist_comp):
+                        yield c
+                    return
             elif child.children[0] == '[':
                 yield testlist_comp
-                continue
-
+                return
         yield child
 
+    if exprlist.type in _STAR_EXPR_PARENTS:
+        for child in exprlist.children[::2]:
+            for c in check_expr(child):  # Python 2 sucks
+                yield c
+    else:
+        for c in check_expr(exprlist):  # Python 2 sucks
+            yield c
+
+
 def _get_expr_stmt_definition_exprs(expr_stmt):
     exprs = []
     for list_ in expr_stmt.children[:-2:2]:
@@ -119,11 +207,24 @@ def _get_expr_stmt_definition_exprs(expr_stmt):
 
 def _get_for_stmt_definition_exprs(for_stmt):
     exprlist = for_stmt.children[1]
-    if exprlist.type != 'exprlist':
-        return [exprlist]
     return list(_iter_definition_exprs_from_lists(exprlist))
 
 
+def _is_argument_comprehension(argument):
+    return argument.children[1].type in _COMP_FOR_TYPES
+
+
+def _any_fstring_error(version, node):
+    if version < (3, 9) or node is None:
+        return False
+    if node.type == "error_node":
+        return any(child.type == "fstring_start" for child in node.children)
+    elif node.type == "fstring":
+        return True
+    else:
+        return search_ancestor(node, "fstring")
+
+
 class _Context(object):
     def __init__(self, node, add_syntax_error, parent_context=None):
         self.node = node
@@ -164,8 +265,7 @@ class _Context(object):
         self._analyze_names(self._global_names, 'global')
         self._analyze_names(self._nonlocal_names, 'nonlocal')
 
-        # Python2.6 doesn't have dict comprehensions.
-        global_name_strs = dict((n.value, n) for n in self._global_names)
+        global_name_strs = {n.value: n for n in self._global_names}
         for nonlocal_name in self._nonlocal_names:
             try:
                 global_name = global_name_strs[nonlocal_name.value]
@@ -273,13 +373,12 @@ class ErrorFinder(Normalizer):
     def visit(self, node):
         if node.type == 'error_node':
             with self.visit_node(node):
-               # Don't need to investigate the inners of an error node. We
-               # might find errors in there that should be ignored, because
-               # the error node itself already shows that there's an issue.
-               return ''
+                # Don't need to investigate the inners of an error node. We
+                # might find errors in there that should be ignored, because
+                # the error node itself already shows that there's an issue.
+                return ''
         return super(ErrorFinder, self).visit(node)
 
-
     @contextmanager
     def visit_node(self, node):
         self._check_type_rules(node)
@@ -323,6 +422,11 @@ class ErrorFinder(Normalizer):
                     match = re.match('\\w{,2}("{1,3}|\'{1,3})', leaf.value)
                     if match is None:
                         message = 'invalid syntax'
+                        if (
+                            self.version >= (3, 9)
+                            and leaf.value in _get_token_collection(self.version).always_break_tokens
+                        ):
+                            message = "f-string: " + message
                     else:
                         if len(match.group(1)) == 1:
                             message = 'EOL while scanning string literal'
@@ -361,8 +465,8 @@ class ErrorFinder(Normalizer):
 class IndentationRule(Rule):
     code = 903
 
-    def _get_message(self, message):
-        message = super(IndentationRule, self)._get_message(message)
+    def _get_message(self, message, node):
+        message = super(IndentationRule, self)._get_message(message, node)
         return "IndentationError: " + message
 
 
@@ -386,21 +490,34 @@ class ErrorFinderConfig(NormalizerConfig):
 class SyntaxRule(Rule):
     code = 901
 
-    def _get_message(self, message):
-        message = super(SyntaxRule, self)._get_message(message)
+    def _get_message(self, message, node):
+        message = super(SyntaxRule, self)._get_message(message, node)
+        if (
+            "f-string" not in message
+            and _any_fstring_error(self._normalizer.version, node)
+        ):
+            message = "f-string: " + message
         return "SyntaxError: " + message
 
 
 @ErrorFinder.register_rule(type='error_node')
 class _InvalidSyntaxRule(SyntaxRule):
     message = "invalid syntax"
+    fstring_message = "f-string: invalid syntax"
 
     def get_node(self, node):
         return node.get_next_leaf()
 
     def is_issue(self, node):
-        # Error leafs will be added later as an error.
-        return node.get_next_leaf().type != 'error_leaf'
+        error = node.get_next_leaf().type != 'error_leaf'
+        if (
+            error
+            and _any_fstring_error(self._normalizer.version, node)
+        ):
+            self.add_issue(node, message=self.fstring_message)
+        else:
+            # Error leafs will be added later as an error.
+            return error
 
 
 @ErrorFinder.register_rule(value='await')
@@ -439,7 +556,11 @@ class _ContinueChecks(SyntaxRule):
                 in_loop = True
             if block.type == 'try_stmt':
                 last_block = block.children[-3]
-                if last_block == 'finally' and leaf.start_pos > last_block.start_pos:
+                if (
+                    last_block == "finally"
+                    and leaf.start_pos > last_block.start_pos
+                    and self._normalizer.version < (3, 8)
+                ):
                     self.add_issue(leaf, message=self.message_in_finally)
                     return False  # Error already added
         if not in_loop:
@@ -455,23 +576,19 @@ class _YieldFromCheck(SyntaxRule):
 
     def is_issue(self, leaf):
         return leaf.parent.type == 'yield_arg' \
-                and self._normalizer.context.is_async_funcdef()
+            and self._normalizer.context.is_async_funcdef()
 
 
 @ErrorFinder.register_rule(type='name')
 class _NameChecks(SyntaxRule):
     message = 'cannot assign to __debug__'
-    message_keyword = 'assignment to keyword'
     message_none = 'cannot assign to None'
 
     def is_issue(self, leaf):
         self._normalizer.context.add_name(leaf)
 
         if leaf.value == '__debug__' and leaf.is_definition():
-            if self._normalizer.version < (3, 0):
-                return True
-            else:
-                self.add_issue(leaf, message=self.message_keyword)
+            return True
         if leaf.value == 'None' and self._normalizer.version < (3, 0) \
                 and leaf.is_definition():
             self.add_issue(leaf, message=self.message_none)
@@ -482,38 +599,38 @@ class _StringChecks(SyntaxRule):
     message = "bytes can only contain ASCII literal characters."
 
     def is_issue(self, leaf):
-            string_prefix = leaf.string_prefix.lower()
-            if 'b' in string_prefix \
-                    and self._normalizer.version >= (3, 0) \
-                    and any(c for c in leaf.value if ord(c) > 127):
-                # b'ä'
-                return True
+        string_prefix = leaf.string_prefix.lower()
+        if 'b' in string_prefix \
+                and self._normalizer.version >= (3, 0) \
+                and any(c for c in leaf.value if ord(c) > 127):
+            # b'ä'
+            return True
 
-            if 'r' not in string_prefix:
-                # Raw strings don't need to be checked if they have proper
-                # escaping.
-                is_bytes = self._normalizer.version < (3, 0)
-                if 'b' in string_prefix:
-                    is_bytes = True
-                if 'u' in string_prefix:
-                    is_bytes = False
+        if 'r' not in string_prefix:
+            # Raw strings don't need to be checked if they have proper
+            # escaping.
+            is_bytes = self._normalizer.version < (3, 0)
+            if 'b' in string_prefix:
+                is_bytes = True
+            if 'u' in string_prefix:
+                is_bytes = False
 
-                payload = leaf._get_payload()
-                if is_bytes:
-                    payload = payload.encode('utf-8')
-                    func = codecs.escape_decode
-                else:
-                    func = codecs.unicode_escape_decode
+            payload = leaf._get_payload()
+            if is_bytes:
+                payload = payload.encode('utf-8')
+                func = codecs.escape_decode
+            else:
+                func = codecs.unicode_escape_decode
 
-                try:
-                    with warnings.catch_warnings():
-                        # The warnings from parsing strings are not relevant.
-                        warnings.filterwarnings('ignore')
-                        func(payload)
-                except UnicodeDecodeError as e:
-                    self.add_issue(leaf, message='(unicode error) ' + str(e))
-                except ValueError as e:
-                    self.add_issue(leaf, message='(value error) ' + str(e))
+            try:
+                with warnings.catch_warnings():
+                    # The warnings from parsing strings are not relevant.
+                    warnings.filterwarnings('ignore')
+                    func(payload)
+            except UnicodeDecodeError as e:
+                self.add_issue(leaf, message='(unicode error) ' + str(e))
+            except ValueError as e:
+                self.add_issue(leaf, message='(value error) ' + str(e))
 
 
 @ErrorFinder.register_rule(value='*')
@@ -539,7 +656,7 @@ class _StarStarCheck(SyntaxRule):
     def is_issue(self, leaf):
         if leaf.parent.type == 'dictorsetmaker':
             comp_for = leaf.get_next_sibling().get_next_sibling()
-            return comp_for is not None and comp_for.type == 'comp_for'
+            return comp_for is not None and comp_for.type in _COMP_FOR_TYPES
 
 
 @ErrorFinder.register_rule(value='yield')
@@ -570,11 +687,14 @@ class _BytesAndStringMix(SyntaxRule):
     message = "cannot mix bytes and nonbytes literals"
 
     def _is_bytes_literal(self, string):
+        if string.type == 'fstring':
+            return False
         return 'b' in string.string_prefix.lower()
 
     def is_issue(self, node):
         first = node.children[0]
-        if first.type == 'string' and self._normalizer.version >= (3, 0):
+        # In Python 2 it's allowed to mix bytes and unicode.
+        if self._normalizer.version >= (3, 0):
             first_is_bytes = self._is_bytes_literal(first)
             for string in node.children[1:]:
                 if first_is_bytes != self._is_bytes_literal(string):
@@ -587,7 +707,7 @@ class _TrailingImportComma(SyntaxRule):
     message = "trailing comma not allowed without surrounding parentheses"
 
     def is_issue(self, node):
-        if node.children[-1] == ',':
+        if node.children[-1] == ',' and node.parent.children[-1] != ')':
             return True
 
 
@@ -613,29 +733,27 @@ class _FutureImportRule(SyntaxRule):
                 allowed_futures = list(ALLOWED_FUTURES)
                 if self._normalizer.version >= (3, 5):
                     allowed_futures.append('generator_stop')
-
+                if self._normalizer.version >= (3, 7):
+                    allowed_futures.append('annotations')
                 if name == 'braces':
-                    self.add_issue(node, message = "not a chance")
+                    self.add_issue(node, message="not a chance")
                 elif name == 'barry_as_FLUFL':
                     m = "Seriously I'm not implementing this :) ~ Dave"
                     self.add_issue(node, message=m)
-                elif name not in ALLOWED_FUTURES:
+                elif name not in allowed_futures:
                     message = "future feature %s is not defined" % name
                     self.add_issue(node, message=message)
 
 
 @ErrorFinder.register_rule(type='star_expr')
 class _StarExprRule(SyntaxRule):
-    message = "starred assignment target must be in a list or tuple"
     message_iterable_unpacking = "iterable unpacking cannot be used in comprehension"
     message_assignment = "can use starred expression only as assignment target"
 
     def is_issue(self, node):
-        if node.parent.type not in _STAR_EXPR_PARENTS:
-            return True
         if node.parent.type == 'testlist_comp':
             # [*[] for a in [1]]
-            if node.parent.children[1].type == 'comp_for':
+            if node.parent.children[1].type in _COMP_FOR_TYPES:
                 self.add_issue(node, message=self.message_iterable_unpacking)
         if self._normalizer.version <= (3, 4):
             n = search_ancestor(node, 'for_stmt', 'expr_stmt')
@@ -656,7 +774,10 @@ class _StarExprRule(SyntaxRule):
 class _StarExprParentRule(SyntaxRule):
     def is_issue(self, node):
         if node.parent.type == 'del_stmt':
-            self.add_issue(node.parent, message="can't use starred expression here")
+            if self._normalizer.version >= (3, 9):
+                self.add_issue(node.parent, message="cannot delete starred")
+            else:
+                self.add_issue(node.parent, message="can't use starred expression here")
         else:
             def is_definition(node, ancestor):
                 if ancestor is None:
@@ -675,7 +796,10 @@ class _StarExprParentRule(SyntaxRule):
                 args = [c for c in node.children if c != ',']
                 starred = [c for c in args if c.type == 'star_expr']
                 if len(starred) > 1:
-                    message = "two starred expressions in assignment"
+                    if self._normalizer.version < (3, 9):
+                        message = "two starred expressions in assignment"
+                    else:
+                        message = "multiple starred expressions in assignment"
                     self.add_issue(starred[1], message=message)
                 elif starred:
                     count = args.index(starred[0])
@@ -712,8 +836,8 @@ class _AnnotatorRule(SyntaxRule):
             if not (lhs.type == 'name'
                     # subscript/attributes are allowed
                     or lhs.type in ('atom_expr', 'power')
-                        and trailer.type == 'trailer'
-                        and trailer.children[0] != '('):
+                    and trailer.type == 'trailer'
+                    and trailer.children[0] != '('):
                 return True
         else:
             # x, y: str
@@ -725,15 +849,27 @@ class _AnnotatorRule(SyntaxRule):
 class _ArgumentRule(SyntaxRule):
     def is_issue(self, node):
         first = node.children[0]
+        if self._normalizer.version < (3, 8):
+            # a((b)=c) is valid in <3.8
+            first = _remove_parens(first)
         if node.children[1] == '=' and first.type != 'name':
             if first.type == 'lambdef':
                 # f(lambda: 1=1)
-                message = "lambda cannot contain assignment"
+                if self._normalizer.version < (3, 8):
+                    message = "lambda cannot contain assignment"
+                else:
+                    message = 'expression cannot contain assignment, perhaps you meant "=="?'
             else:
                 # f(+x=1)
-                message = "keyword can't be an expression"
+                if self._normalizer.version < (3, 8):
+                    message = "keyword can't be an expression"
+                else:
+                    message = 'expression cannot contain assignment, perhaps you meant "=="?'
             self.add_issue(first, message=message)
 
+        if _is_argument_comprehension(node) and node.parent.type == 'classdef':
+            self.add_issue(node, message='invalid syntax')
+
 
 @ErrorFinder.register_rule(type='nonlocal_stmt')
 class _NonlocalModuleLevelRule(SyntaxRule):
@@ -753,58 +889,61 @@ class _ArglistRule(SyntaxRule):
             return "Generator expression must be parenthesized"
 
     def is_issue(self, node):
-        first_arg = node.children[0]
-        if first_arg.type == 'argument' \
-                and first_arg.children[1].type == 'comp_for':
-            # e.g. foo(x for x in [], b)
-            return len(node.children) >= 2
-        else:
-            arg_set = set()
-            kw_only = False
-            kw_unpacking_only = False
-            is_old_starred = False
-            # In python 3 this would be a bit easier (stars are part of
-            # argument), but we have to understand both.
-            for argument in node.children:
-                if argument == ',':
-                    continue
+        arg_set = set()
+        kw_only = False
+        kw_unpacking_only = False
+        is_old_starred = False
+        # In python 3 this would be a bit easier (stars are part of
+        # argument), but we have to understand both.
+        for argument in node.children:
+            if argument == ',':
+                continue
 
-                if argument in ('*', '**'):
-                    # Python < 3.5 has the order engraved in the grammar
-                    # file.  No need to do anything here.
-                    is_old_starred = True
-                    continue
-                if is_old_starred:
-                    is_old_starred = False
-                    continue
+            if argument in ('*', '**'):
+                # Python < 3.5 has the order engraved in the grammar
+                # file.  No need to do anything here.
+                is_old_starred = True
+                continue
+            if is_old_starred:
+                is_old_starred = False
+                continue
 
-                if argument.type == 'argument':
-                    first = argument.children[0]
-                    if first in ('*', '**'):
-                        if first == '*':
-                            if kw_unpacking_only:
-                                # foo(**kwargs, *args)
-                                message = "iterable argument unpacking follows keyword argument unpacking"
-                                self.add_issue(argument, message=message)
+            if argument.type == 'argument':
+                first = argument.children[0]
+                if _is_argument_comprehension(argument) and len(node.children) >= 2:
+                    # a(a, b for b in c)
+                    return True
+
+                if first in ('*', '**'):
+                    if first == '*':
+                        if kw_unpacking_only:
+                            # foo(**kwargs, *args)
+                            message = "iterable argument unpacking " \
+                                      "follows keyword argument unpacking"
+                            self.add_issue(argument, message=message)
+                    else:
+                        kw_unpacking_only = True
+                else:  # Is a keyword argument.
+                    kw_only = True
+                    if first.type == 'name':
+                        if first.value in arg_set:
+                            # f(x=1, x=2)
+                            message = "keyword argument repeated"
+                            if self._normalizer.version >= (3, 9):
+                                message += ": {}".format(first.value)
+                            self.add_issue(first, message=message)
                         else:
-                            kw_unpacking_only = True
-                    else:  # Is a keyword argument.
-                        kw_only = True
-                        if first.type == 'name':
-                            if first.value in arg_set:
-                                # f(x=1, x=2)
-                                self.add_issue(first, message="keyword argument repeated")
-                            else:
-                                arg_set.add(first.value)
-                else:
-                    if kw_unpacking_only:
-                        # f(**x, y)
-                        message = "positional argument follows keyword argument unpacking"
-                        self.add_issue(argument, message=message)
-                    elif kw_only:
-                        # f(x=2, y)
-                        message = "positional argument follows keyword argument"
-                        self.add_issue(argument, message=message)
+                            arg_set.add(first.value)
+            else:
+                if kw_unpacking_only:
+                    # f(**x, y)
+                    message = "positional argument follows keyword argument unpacking"
+                    self.add_issue(argument, message=message)
+                elif kw_only:
+                    # f(x=2, y)
+                    message = "positional argument follows keyword argument"
+                    self.add_issue(argument, message=message)
+
 
 @ErrorFinder.register_rule(type='parameters')
 @ErrorFinder.register_rule(type='lambdef')
@@ -846,6 +985,7 @@ class _TryStmtRule(SyntaxRule):
 @ErrorFinder.register_rule(type='fstring')
 class _FStringRule(SyntaxRule):
     _fstring_grammar = None
+    message_expr = "f-string expression part cannot include a backslash"
     message_nested = "f-string: expressions nested too deeply"
     message_conversion = "f-string: invalid conversion character: expected 's', 'r', or 'a'"
 
@@ -856,6 +996,10 @@ class _FStringRule(SyntaxRule):
         if depth >= 2:
             self.add_issue(fstring_expr, message=self.message_nested)
 
+        expr = fstring_expr.children[1]
+        if '\\' in expr.get_code():
+            self.add_issue(expr, message=self.message_expr)
+
         conversion = fstring_expr.children[2]
         if conversion.type == 'fstring_conversion':
             name = conversion.children[1]
@@ -876,7 +1020,7 @@ class _FStringRule(SyntaxRule):
 
 
 class _CheckAssignmentRule(SyntaxRule):
-    def _check_assignment(self, node, is_deletion=False):
+    def _check_assignment(self, node, is_deletion=False, is_namedexpr=False, is_aug_assign=False):
         error = None
         type_ = node.type
         if type_ == 'lambdef':
@@ -886,19 +1030,48 @@ class _CheckAssignmentRule(SyntaxRule):
             error = _get_comprehension_type(node)
             if error is None:
                 if second.type == 'dictorsetmaker':
-                    error = 'literal'
+                    if self._normalizer.version < (3, 8):
+                        error = 'literal'
+                    else:
+                        if second.children[1] == ':':
+                            error = 'dict display'
+                        else:
+                            error = 'set display'
+                elif first == "{" and second == "}":
+                    if self._normalizer.version < (3, 8):
+                        error = 'literal'
+                    else:
+                        error = "dict display"
+                elif first == "{" and len(node.children) > 2:
+                    if self._normalizer.version < (3, 8):
+                        error = 'literal'
+                    else:
+                        error = "set display"
                 elif first in ('(', '['):
                     if second.type == 'yield_expr':
                         error = 'yield expression'
                     elif second.type == 'testlist_comp':
+                        # ([a, b] := [1, 2])
+                        # ((a, b) := [1, 2])
+                        if is_namedexpr:
+                            if first == '(':
+                                error = 'tuple'
+                            elif first == '[':
+                                error = 'list'
+
                         # This is not a comprehension, they were handled
                         # further above.
                         for child in second.children[::2]:
-                            self._check_assignment(child, is_deletion)
+                            self._check_assignment(child, is_deletion, is_namedexpr, is_aug_assign)
                     else:  # Everything handled, must be useless brackets.
-                        self._check_assignment(second, is_deletion)
+                        self._check_assignment(second, is_deletion, is_namedexpr, is_aug_assign)
         elif type_ == 'keyword':
-            error = 'keyword'
+            if node.value == "yield":
+                error = "yield expression"
+            elif self._normalizer.version < (3, 8):
+                error = 'keyword'
+            else:
+                error = str(node.value)
         elif type_ == 'operator':
             if node.value == '...':
                 error = 'Ellipsis'
@@ -923,45 +1096,87 @@ class _CheckAssignmentRule(SyntaxRule):
                 assert trailer.type == 'trailer'
                 if trailer.children[0] == '(':
                     error = 'function call'
+                elif is_namedexpr and trailer.children[0] == '[':
+                    error = 'subscript'
+                elif is_namedexpr and trailer.children[0] == '.':
+                    error = 'attribute'
+        elif type_ == "fstring":
+            if self._normalizer.version < (3, 8):
+                error = 'literal'
+            else:
+                error = "f-string expression"
         elif type_ in ('testlist_star_expr', 'exprlist', 'testlist'):
             for child in node.children[::2]:
-                self._check_assignment(child, is_deletion)
-        elif ('expr' in type_ and type_ != 'star_expr' # is a substring
+                self._check_assignment(child, is_deletion, is_namedexpr, is_aug_assign)
+        elif ('expr' in type_ and type_ != 'star_expr'  # is a substring
               or '_test' in type_
               or type_ in ('term', 'factor')):
             error = 'operator'
+        elif type_ == "star_expr":
+            if is_deletion:
+                if self._normalizer.version >= (3, 9):
+                    error = "starred"
+                else:
+                    self.add_issue(node, message="can't use starred expression here")
+            elif not search_ancestor(node, *_STAR_EXPR_PARENTS) and not is_aug_assign:
+                self.add_issue(node, message="starred assignment target must be in a list or tuple")
+
+            self._check_assignment(node.children[1])
 
         if error is not None:
-            message = "can't %s %s" % ("delete" if is_deletion else "assign to", error)
+            if is_namedexpr:
+                message = 'cannot use assignment expressions with %s' % error
+            else:
+                cannot = "can't" if self._normalizer.version < (3, 8) else "cannot"
+                message = ' '.join([cannot, "delete" if is_deletion else "assign to", error])
             self.add_issue(node, message=message)
 
 
-@ErrorFinder.register_rule(type='comp_for')
+@ErrorFinder.register_rule(type='sync_comp_for')
 class _CompForRule(_CheckAssignmentRule):
     message = "asynchronous comprehension outside of an asynchronous function"
 
     def is_issue(self, node):
-        # Some of the nodes here are already used, so no else if
-        expr_list = node.children[1 + int(node.children[0] == 'async')]
+        expr_list = node.children[1]
         if expr_list.type != 'expr_list':  # Already handled.
             self._check_assignment(expr_list)
 
-        return node.children[0] == 'async' \
+        return node.parent.children[0] == 'async' \
             and not self._normalizer.context.is_async_funcdef()
 
 
 @ErrorFinder.register_rule(type='expr_stmt')
 class _ExprStmtRule(_CheckAssignmentRule):
     message = "illegal expression for augmented assignment"
-
+    extended_message = "'{target}' is an " + message
     def is_issue(self, node):
-        for before_equal in node.children[:-2:2]:
-            self._check_assignment(before_equal)
-
         augassign = node.children[1]
-        if augassign != '=' and augassign.type != 'annassign':  # Is augassign.
-            return node.children[0].type in ('testlist_star_expr', 'atom', 'testlist')
+        is_aug_assign = augassign != '=' and augassign.type != 'annassign'
 
+        if self._normalizer.version <= (3, 8) or not is_aug_assign:
+            for before_equal in node.children[:-2:2]:
+                self._check_assignment(before_equal, is_aug_assign=is_aug_assign)
+
+        if is_aug_assign:
+            target = _remove_parens(node.children[0])
+            # a, a[b], a.b
+
+            if target.type == "name" or (
+                target.type in ("atom_expr", "power")
+                and target.children[1].type == "trailer"
+                and target.children[-1].children[0] != "("
+            ):
+                return False
+
+            if self._normalizer.version <= (3, 8):
+                return True
+            else:
+                self.add_issue(
+                    node,
+                    message=self.extended_message.format(
+                        target=_get_rhs_name(node.children[0], self._normalizer.version)
+                    ),
+                )
 
 @ErrorFinder.register_rule(type='with_item')
 class _WithItemRule(_CheckAssignmentRule):
@@ -992,3 +1207,71 @@ class _ForStmtRule(_CheckAssignmentRule):
         expr_list = for_stmt.children[1]
         if expr_list.type != 'expr_list':  # Already handled.
             self._check_assignment(expr_list)
+
+
+@ErrorFinder.register_rule(type='namedexpr_test')
+class _NamedExprRule(_CheckAssignmentRule):
+    # namedexpr_test: test [':=' test]
+
+    def is_issue(self, namedexpr_test):
+        # assigned name
+        first = namedexpr_test.children[0]
+
+        def search_namedexpr_in_comp_for(node):
+            while True:
+                parent = node.parent
+                if parent is None:
+                    return parent
+                if parent.type == 'sync_comp_for' and parent.children[3] == node:
+                    return parent
+                node = parent
+
+        if search_namedexpr_in_comp_for(namedexpr_test):
+            # [i+1 for i in (i := range(5))]
+            # [i+1 for i in (j := range(5))]
+            # [i+1 for i in (lambda: (j := range(5)))()]
+            message = 'assignment expression cannot be used in a comprehension iterable expression'
+            self.add_issue(namedexpr_test, message=message)
+
+        # defined names
+        exprlist = list()
+
+        def process_comp_for(comp_for):
+            if comp_for.type == 'sync_comp_for':
+                comp = comp_for
+            elif comp_for.type == 'comp_for':
+                comp = comp_for.children[1]
+            exprlist.extend(_get_for_stmt_definition_exprs(comp))
+
+        def search_all_comp_ancestors(node):
+            has_ancestors = False
+            while True:
+                node = search_ancestor(node, 'testlist_comp', 'dictorsetmaker')
+                if node is None:
+                    break
+                for child in node.children:
+                    if child.type in _COMP_FOR_TYPES:
+                        process_comp_for(child)
+                        has_ancestors = True
+                        break
+            return has_ancestors
+
+        # check assignment expressions in comprehensions
+        search_all = search_all_comp_ancestors(namedexpr_test)
+        if search_all:
+            if self._normalizer.context.node.type == 'classdef':
+                message = 'assignment expression within a comprehension ' \
+                          'cannot be used in a class body'
+                self.add_issue(namedexpr_test, message=message)
+
+            namelist = [expr.value for expr in exprlist if expr.type == 'name']
+            if first.type == 'name' and first.value in namelist:
+                # [i := 0 for i, j in range(5)]
+                # [[(i := i) for j in range(5)] for i in range(5)]
+                # [i for i, j in range(5) if True or (i := 1)]
+                # [False and (i := 0) for i, j in range(5)]
+                message = 'assignment expression cannot rebind ' \
+                          'comprehension iteration variable %r' % first.value
+                self.add_issue(namedexpr_test, message=message)
+
+        self._check_assignment(first, is_namedexpr=True)

parso/python/grammar26.txt

@@ -1,159 +0,0 @@
-# Grammar for Python
-
-# Note:  Changing the grammar specified in this file will most likely
-#        require corresponding changes in the parser module
-#        (../Modules/parsermodule.c).  If you can't make the changes to
-#        that module yourself, please co-ordinate the required changes
-#        with someone who can; ask around on python-dev for help.  Fred
-#        Drake <fdrake@acm.org> will probably be listening there.
-
-# NOTE WELL: You should also follow all the steps listed in PEP 306,
-# "How to Change Python's Grammar"
-
-# Commands for Kees Blom's railroad program
-#diagram:token NAME
-#diagram:token NUMBER
-#diagram:token STRING
-#diagram:token NEWLINE
-#diagram:token ENDMARKER
-#diagram:token INDENT
-#diagram:output\input python.bla
-#diagram:token DEDENT
-#diagram:output\textwidth 20.04cm\oddsidemargin  0.0cm\evensidemargin 0.0cm
-#diagram:rules
-
-# Start symbols for the grammar:
-#	single_input is a single interactive statement;
-#	file_input is a module or sequence of commands read from an input file;
-#	eval_input is the input for the eval() and input() functions.
-# NB: compound_stmt in single_input is followed by extra NEWLINE!
-single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
-eval_input: testlist NEWLINE* ENDMARKER
-
-decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
-decorators: decorator+
-decorated: decorators (classdef | funcdef)
-funcdef: 'def' NAME parameters ':' suite
-parameters: '(' [varargslist] ')'
-varargslist: ((fpdef ['=' test] ',')*
-              ('*' NAME [',' '**' NAME] | '**' NAME) |
-              fpdef ['=' test] (',' fpdef ['=' test])* [','])
-fpdef: NAME | '(' fplist ')'
-fplist: fpdef (',' fpdef)* [',']
-
-stmt: simple_stmt | compound_stmt
-simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
-small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
-             import_stmt | global_stmt | exec_stmt | assert_stmt)
-expr_stmt: testlist (augassign (yield_expr|testlist) |
-                     ('=' (yield_expr|testlist))*)
-augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
-            '<<=' | '>>=' | '**=' | '//=')
-# For normal assignments, additional restrictions enforced by the interpreter
-print_stmt: 'print' ( [ test (',' test)* [','] ] |
-                      '>>' test [ (',' test)+ [','] ] )
-del_stmt: 'del' exprlist
-pass_stmt: 'pass'
-flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
-break_stmt: 'break'
-continue_stmt: 'continue'
-return_stmt: 'return' [testlist]
-yield_stmt: yield_expr
-raise_stmt: 'raise' [test [',' test [',' test]]]
-import_stmt: import_name | import_from
-import_name: 'import' dotted_as_names
-import_from: ('from' ('.'* dotted_name | '.'+)
-              'import' ('*' | '(' import_as_names ')' | import_as_names))
-import_as_name: NAME ['as' NAME]
-dotted_as_name: dotted_name ['as' NAME]
-import_as_names: import_as_name (',' import_as_name)* [',']
-dotted_as_names: dotted_as_name (',' dotted_as_name)*
-dotted_name: NAME ('.' NAME)*
-global_stmt: 'global' NAME (',' NAME)*
-exec_stmt: 'exec' expr ['in' test [',' test]]
-assert_stmt: 'assert' test [',' test]
-
-compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
-if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
-while_stmt: 'while' test ':' suite ['else' ':' suite]
-for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
-try_stmt: ('try' ':' suite
-           ((except_clause ':' suite)+
-	    ['else' ':' suite]
-	    ['finally' ':' suite] |
-	   'finally' ':' suite))
-with_stmt: 'with' with_item ':' suite
-# Dave: Python2.6 actually defines a little bit of a different label called
-#       'with_var'. However in 2.7+ this is the default. Apply it for
-#       consistency reasons.
-with_item: test ['as' expr]
-# NB compile.c makes sure that the default except clause is last
-except_clause: 'except' [test [('as' | ',') test]]
-suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
-
-# Backward compatibility cruft to support:
-# [ x for x in lambda: True, lambda: False if x() ]
-# even while also allowing:
-# lambda x: 5 if x else 2
-# (But not a mix of the two)
-testlist_safe: old_test [(',' old_test)+ [',']]
-old_test: or_test | old_lambdef
-old_lambdef: 'lambda' [varargslist] ':' old_test
-
-test: or_test ['if' or_test 'else' test] | lambdef
-or_test: and_test ('or' and_test)*
-and_test: not_test ('and' not_test)*
-not_test: 'not' not_test | comparison
-comparison: expr (comp_op expr)*
-comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
-expr: xor_expr ('|' xor_expr)*
-xor_expr: and_expr ('^' and_expr)*
-and_expr: shift_expr ('&' shift_expr)*
-shift_expr: arith_expr (('<<'|'>>') arith_expr)*
-arith_expr: term (('+'|'-') term)*
-term: factor (('*'|'/'|'%'|'//') factor)*
-factor: ('+'|'-'|'~') factor | power
-power: atom trailer* ['**' factor]
-atom: ('(' [yield_expr|testlist_comp] ')' |
-       '[' [listmaker] ']' |
-       '{' [dictorsetmaker] '}' |
-       '`' testlist1 '`' |
-       NAME | NUMBER | strings)
-strings: STRING+
-listmaker: test ( list_for | (',' test)* [','] )
-# Dave: Renamed testlist_gexpr to testlist_comp, because in 2.7+ this is the
-#       default. It's more consistent like this.
-testlist_comp: test ( gen_for | (',' test)* [','] )
-lambdef: 'lambda' [varargslist] ':' test
-trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
-subscriptlist: subscript (',' subscript)* [',']
-subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
-sliceop: ':' [test]
-exprlist: expr (',' expr)* [',']
-testlist: test (',' test)* [',']
-# Dave: Rename from dictmaker to dictorsetmaker, because this is more
-#       consistent with the following grammars.
-dictorsetmaker: test ':' test (',' test ':' test)* [',']
-
-classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
-
-arglist: (argument ',')* (argument [',']
-                         |'*' test (',' argument)* [',' '**' test] 
-                         |'**' test)
-argument: test [gen_for] | test '=' test  # Really [keyword '='] test
-
-list_iter: list_for | list_if
-list_for: 'for' exprlist 'in' testlist_safe [list_iter]
-list_if: 'if' old_test [list_iter]
-
-gen_iter: gen_for | gen_if
-gen_for: 'for' exprlist 'in' or_test [gen_iter]
-gen_if: 'if' old_test [gen_iter]
-
-testlist1: test (',' test)*
-
-# not used in grammar, but may appear in "node" passed from Parser to Compiler
-encoding_decl: NAME
-
-yield_expr: 'yield' [testlist]

parso/python/grammar27.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() and input() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -30,7 +30,7 @@ varargslist: ((fpdef ['=' test] ',')*
 fpdef: NAME | '(' fplist ')'
 fplist: fpdef (',' fpdef)* [',']
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | print_stmt  | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | exec_stmt | assert_stmt)
@@ -107,7 +107,7 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        NAME | NUMBER | strings)
 strings: STRING+
 listmaker: test ( list_for | (',' test)* [','] )
-testlist_comp: test ( comp_for | (',' test)* [','] )
+testlist_comp: test ( sync_comp_for | (',' test)* [','] )
 lambdef: 'lambda' [varargslist] ':' test
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
@@ -115,8 +115,8 @@ subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: expr (',' expr)* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                  (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( (test ':' test (sync_comp_for | (',' test ':' test)* [','])) |
+                  (test (sync_comp_for | (',' test)* [','])) )
 
 classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
 
@@ -125,14 +125,14 @@ arglist: (argument ',')* (argument [',']
                          |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test
+argument: test [sync_comp_for] | test '=' test
 
 list_iter: list_for | list_if
 list_for: 'for' exprlist 'in' testlist_safe [list_iter]
 list_if: 'if' old_test [list_iter]
 
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' old_test [comp_iter]
 
 testlist1: test (',' test)*

parso/python/grammar310.txt

@@ -0,0 +1,171 @@
+# Grammar for Python
+
+# NOTE WELL: You should also follow all the steps listed at
+# https://devguide.python.org/grammar/
+
+# Start symbols for the grammar:
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+file_input: stmt* ENDMARKER
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' namedexpr_test NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+async_funcdef: 'async' funcdef
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+
+parameters: '(' [typedargslist] ')'
+typedargslist: (
+  (tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [',' [ tfpdef ['=' test] (
+        ',' tfpdef ['=' test])* ([',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]])
+  | '*' [tfpdef] (',' tfpdef ['=' test])* ([',' ['**' tfpdef [',']]])
+  | '**' tfpdef [',']]] )
+|  (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]]
+  | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+  | '**' tfpdef [','])
+)
+tfpdef: NAME [':' test]
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']
+)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt | NEWLINE
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal and annotated assignments, additional restrictions enforced by the interpreter
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist_star_expr]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test ['from' test]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test ['as' NAME]]
+suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
+
+namedexpr_test: test [':=' test]
+test: or_test ['if' or_test 'else' test] | lambdef
+test_nocond: or_test | lambdef_nocond
+lambdef: 'lambda' [varargslist] ':' test
+lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401 (which really works :-)
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'@'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
+testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test | star_expr)
+                   (comp_for | (',' (test | star_expr))* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: argument (',' argument)*  [',']
+
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguments are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test ':=' test |
+            test '=' test |
+            '**' test |
+            '*' test )
+
+comp_iter: comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
+comp_if: 'if' test_nocond [comp_iter]
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist_star_expr
+
+strings: (STRING | fstring)+
+fstring: FSTRING_START fstring_content* FSTRING_END
+fstring_content: FSTRING_STRING | fstring_expr
+fstring_conversion: '!' NAME
+fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_format_spec: ':' fstring_content*

parso/python/grammar33.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -33,7 +33,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
      |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
@@ -105,15 +105,15 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
 strings: STRING+
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (test|star_expr) ( sync_comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                  (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( (test ':' test (sync_comp_for | (',' test ':' test)* [','])) |
+                  (test (sync_comp_for | (',' test)* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -122,9 +122,9 @@ arglist: (argument ',')* (argument [',']
                          |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+argument: test [sync_comp_for] | test '=' test  # Really [keyword '='] test
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar34.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -33,7 +33,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
      |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
@@ -105,15 +105,15 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
 strings: STRING+
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (test|star_expr) ( sync_comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                  (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( (test ':' test (sync_comp_for | (',' test ':' test)* [','])) |
+                  (test (sync_comp_for | (',' test)* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -122,9 +122,9 @@ arglist: (argument ',')* (argument [',']
                          |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+argument: test [sync_comp_for] | test '=' test  # Really [keyword '='] test
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar35.txt

@@ -16,7 +16,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -38,7 +38,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [','
      |  '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef)
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
@@ -112,7 +112,7 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
 strings: STRING+
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (test|star_expr) ( sync_comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
@@ -120,9 +120,9 @@ sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
 dictorsetmaker: ( ((test ':' test | '**' expr)
-                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                   (sync_comp_for | (',' (test ':' test | '**' expr))* [','])) |
                   ((test | star_expr)
-                   (comp_for | (',' (test | star_expr))* [','])) )
+                   (sync_comp_for | (',' (test | star_expr))* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -137,13 +137,13 @@ arglist: argument (',' argument)*  [',']
 # Illegal combinations and orderings are blocked in ast.c:
 # multiple (test comp_for) arguments are blocked; keyword unpackings
 # that precede iterable unpackings are blocked; etc.
-argument: ( test [comp_for] |
+argument: ( test [sync_comp_for] |
             test '=' test |
             '**' test |
             '*' test )
 
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar36.txt

@@ -9,7 +9,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
@@ -35,7 +35,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
 )
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
@@ -140,7 +140,8 @@ argument: ( test [comp_for] |
             '*' test )
 
 comp_iter: comp_for | comp_if
-comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar37.txt

@@ -9,7 +9,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
 decorators: decorator+
@@ -33,7 +33,7 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
 )
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
@@ -138,7 +138,8 @@ argument: ( test [comp_for] |
             '*' test )
 
 comp_iter: comp_for | comp_if
-comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar38.txt

@@ -9,7 +9,7 @@
 #       eval_input is the input for the eval() functions.
 # NB: compound_stmt in single_input is followed by extra NEWLINE!
 single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
-file_input: (NEWLINE | stmt)* ENDMARKER
+file_input: stmt* ENDMARKER
 eval_input: testlist NEWLINE* ENDMARKER
 
 decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
@@ -20,13 +20,25 @@ async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 
 parameters: '(' [typedargslist] ')'
-typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+typedargslist: (
+  (tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [',' [ tfpdef ['=' test] (
+        ',' tfpdef ['=' test])* ([',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]])
+  | '*' [tfpdef] (',' tfpdef ['=' test])* ([',' ['**' tfpdef [',']]])
+  | '**' tfpdef [',']]] )
+|  (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
         '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
       | '**' tfpdef [',']]]
   | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
   | '**' tfpdef [','])
+)
 tfpdef: NAME [':' test]
-varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
         '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
       | '**' vfpdef [',']]]
   | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
@@ -34,13 +46,13 @@ varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
 )
 vfpdef: NAME
 
-stmt: simple_stmt | compound_stmt
+stmt: simple_stmt | compound_stmt | NEWLINE
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
              import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
 expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist_star_expr))*)
-annassign: ':' test ['=' test]
+annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
 testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
 augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
             '<<=' | '>>=' | '**=' | '//=')
@@ -69,8 +81,8 @@ assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
 async_stmt: 'async' (funcdef | with_stmt | for_stmt)
-if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
-while_stmt: 'while' test ':' suite ['else' ':' suite]
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
 try_stmt: ('try' ':' suite
            ((except_clause ':' suite)+
@@ -83,6 +95,7 @@ with_item: test ['as' expr]
 except_clause: 'except' [test ['as' NAME]]
 suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
 
+namedexpr_test: test [':=' test]
 test: or_test ['if' or_test 'else' test] | lambdef
 test_nocond: or_test | lambdef_nocond
 lambdef: 'lambda' [varargslist] ':' test
@@ -108,7 +121,7 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
@@ -134,6 +147,7 @@ arglist: argument (',' argument)*  [',']
 # multiple (test comp_for) arguments are blocked; keyword unpackings
 # that precede iterable unpackings are blocked; etc.
 argument: ( test [comp_for] |
+            test ':=' test |
             test '=' test |
             '**' test |
             '*' test )
@@ -153,5 +167,5 @@ strings: (STRING | fstring)+
 fstring: FSTRING_START fstring_content* FSTRING_END
 fstring_content: FSTRING_STRING | fstring_expr
 fstring_conversion: '!' NAME
-fstring_expr: '{' testlist [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
 fstring_format_spec: ':' fstring_content*

parso/python/grammar39.txt

@@ -0,0 +1,171 @@
+# Grammar for Python
+
+# NOTE WELL: You should also follow all the steps listed at
+# https://devguide.python.org/grammar/
+
+# Start symbols for the grammar:
+#       single_input is a single interactive statement;
+#       file_input is a module or sequence of commands read from an input file;
+#       eval_input is the input for the eval() functions.
+# NB: compound_stmt in single_input is followed by extra NEWLINE!
+single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
+file_input: stmt* ENDMARKER
+eval_input: testlist NEWLINE* ENDMARKER
+
+decorator: '@' namedexpr_test NEWLINE
+decorators: decorator+
+decorated: decorators (classdef | funcdef | async_funcdef)
+
+async_funcdef: 'async' funcdef
+funcdef: 'def' NAME parameters ['->' test] ':' suite
+
+parameters: '(' [typedargslist] ')'
+typedargslist: (
+  (tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [',' [ tfpdef ['=' test] (
+        ',' tfpdef ['=' test])* ([',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]])
+  | '*' [tfpdef] (',' tfpdef ['=' test])* ([',' ['**' tfpdef [',']]])
+  | '**' tfpdef [',']]] )
+|  (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]]
+  | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+  | '**' tfpdef [','])
+)
+tfpdef: NAME [':' test]
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']
+)
+vfpdef: NAME
+
+stmt: simple_stmt | compound_stmt | NEWLINE
+simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
+small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
+expr_stmt: testlist_star_expr (annassign | augassign (yield_expr|testlist) |
+                     ('=' (yield_expr|testlist_star_expr))*)
+annassign: ':' test ['=' (yield_expr|testlist_star_expr)]
+testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
+augassign: ('+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' |
+            '<<=' | '>>=' | '**=' | '//=')
+# For normal and annotated assignments, additional restrictions enforced by the interpreter
+del_stmt: 'del' exprlist
+pass_stmt: 'pass'
+flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
+break_stmt: 'break'
+continue_stmt: 'continue'
+return_stmt: 'return' [testlist_star_expr]
+yield_stmt: yield_expr
+raise_stmt: 'raise' [test ['from' test]]
+import_stmt: import_name | import_from
+import_name: 'import' dotted_as_names
+# note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
+import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
+              'import' ('*' | '(' import_as_names ')' | import_as_names))
+import_as_name: NAME ['as' NAME]
+dotted_as_name: dotted_name ['as' NAME]
+import_as_names: import_as_name (',' import_as_name)* [',']
+dotted_as_names: dotted_as_name (',' dotted_as_name)*
+dotted_name: NAME ('.' NAME)*
+global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
+assert_stmt: 'assert' test [',' test]
+
+compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
+async_stmt: 'async' (funcdef | with_stmt | for_stmt)
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
+for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
+try_stmt: ('try' ':' suite
+           ((except_clause ':' suite)+
+            ['else' ':' suite]
+            ['finally' ':' suite] |
+           'finally' ':' suite))
+with_stmt: 'with' with_item (',' with_item)*  ':' suite
+with_item: test ['as' expr]
+# NB compile.c makes sure that the default except clause is last
+except_clause: 'except' [test ['as' NAME]]
+suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
+
+namedexpr_test: test [':=' test]
+test: or_test ['if' or_test 'else' test] | lambdef
+test_nocond: or_test | lambdef_nocond
+lambdef: 'lambda' [varargslist] ':' test
+lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
+or_test: and_test ('or' and_test)*
+and_test: not_test ('and' not_test)*
+not_test: 'not' not_test | comparison
+comparison: expr (comp_op expr)*
+# <> isn't actually a valid comparison operator in Python. It's here for the
+# sake of a __future__ import described in PEP 401 (which really works :-)
+comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
+star_expr: '*' expr
+expr: xor_expr ('|' xor_expr)*
+xor_expr: and_expr ('^' and_expr)*
+and_expr: shift_expr ('&' shift_expr)*
+shift_expr: arith_expr (('<<'|'>>') arith_expr)*
+arith_expr: term (('+'|'-') term)*
+term: factor (('*'|'@'|'/'|'%'|'//') factor)*
+factor: ('+'|'-'|'~') factor | power
+power: atom_expr ['**' factor]
+atom_expr: ['await'] atom trailer*
+atom: ('(' [yield_expr|testlist_comp] ')' |
+       '[' [testlist_comp] ']' |
+       '{' [dictorsetmaker] '}' |
+       NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
+testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
+trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
+subscriptlist: subscript (',' subscript)* [',']
+subscript: test | [test] ':' [test] [sliceop]
+sliceop: ':' [test]
+exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
+testlist: test (',' test)* [',']
+dictorsetmaker: ( ((test ':' test | '**' expr)
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test | star_expr)
+                   (comp_for | (',' (test | star_expr))* [','])) )
+
+classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
+
+arglist: argument (',' argument)*  [',']
+
+# The reason that keywords are test nodes instead of NAME is that using NAME
+# results in an ambiguity. ast.c makes sure it's a NAME.
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguments are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test ':=' test |
+            test '=' test |
+            '**' test |
+            '*' test )
+
+comp_iter: comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
+comp_if: 'if' test_nocond [comp_iter]
+
+# not used in grammar, but may appear in "node" passed from Parser to Compiler
+encoding_decl: NAME
+
+yield_expr: 'yield' [yield_arg]
+yield_arg: 'from' test | testlist_star_expr
+
+strings: (STRING | fstring)+
+fstring: FSTRING_START fstring_content* FSTRING_END
+fstring_content: FSTRING_STRING | fstring_expr
+fstring_conversion: '!' NAME
+fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_format_spec: ':' fstring_content*

parso/python/issue_list.txt

@@ -172,5 +172,5 @@ A list of syntax/indentation errors I've encountered in CPython.
 Version specific:
 Python 3.5:
     'yield' inside async function
-Python 3.3/3.4:
+Python 3.4:
     can use starred expression only as assignment target

parso/python/parser.py

@@ -39,13 +39,11 @@ class Parser(BaseParser):
         'for_stmt': tree.ForStmt,
         'while_stmt': tree.WhileStmt,
         'try_stmt': tree.TryStmt,
-        'comp_for': tree.CompFor,
+        'sync_comp_for': tree.SyncCompFor,
         # Not sure if this is the best idea, but IMO it's the easiest way to
         # avoid extreme amounts of work around the subtle difference of 2/3
         # grammar in list comoprehensions.
-        'list_for': tree.CompFor,
-        # Same here. This just exists in Python 2.6.
-        'gen_for': tree.CompFor,
+        'list_for': tree.SyncCompFor,
         'decorator': tree.Decorator,
         'lambdef': tree.Lambda,
         'old_lambdef': tree.Lambda,
@@ -128,10 +126,10 @@ class Parser(BaseParser):
 
         if self._start_nonterminal == 'file_input' and \
                 (token.type == PythonTokenTypes.ENDMARKER
-                 or token.type == DEDENT and '\n' not in last_leaf.value
-                 and '\r' not in last_leaf.value):
+                 or token.type == DEDENT and not last_leaf.value.endswith('\n')
+                 and not last_leaf.value.endswith('\r')):
             # In Python statements need to end with a newline. But since it's
-            # possible (and valid in Python ) that there's no newline at the
+            # possible (and valid in Python) that there's no newline at the
             # end of a file, we have to recover even if the user doesn't want
             # error recovery.
             if self.stack[-1].dfa.from_rule == 'simple_stmt':
@@ -210,6 +208,7 @@ class Parser(BaseParser):
                 o = self._omit_dedent_list
                 if o and o[-1] == self._indent_counter:
                     o.pop()
+                    self._indent_counter -= 1
                     continue
 
                 self._indent_counter -= 1

parso/python/token.pyi

@@ -0,0 +1,30 @@
+from typing import Container, Iterable
+
+class TokenType:
+    name: str
+    contains_syntax: bool
+    def __init__(self, name: str, contains_syntax: bool) -> None: ...
+
+class TokenTypes:
+    def __init__(
+        self, names: Iterable[str], contains_syntax: Container[str]
+    ) -> None: ...
+
+# not an actual class in the source code, but we need this class to type the fields of
+# PythonTokenTypes
+class _FakePythonTokenTypesClass(TokenTypes):
+    STRING: TokenType
+    NUMBER: TokenType
+    NAME: TokenType
+    ERRORTOKEN: TokenType
+    NEWLINE: TokenType
+    INDENT: TokenType
+    DEDENT: TokenType
+    ERROR_DEDENT: TokenType
+    FSTRING_STRING: TokenType
+    FSTRING_START: TokenType
+    FSTRING_END: TokenType
+    OP: TokenType
+    ENDMARKER: TokenType
+
+PythonTokenTypes: _FakePythonTokenTypesClass = ...

parso/python/tokenize.py

@@ -12,17 +12,18 @@ memory optimizations here.
 from __future__ import absolute_import
 
 import sys
-import string
 import re
 from collections import namedtuple
 import itertools as _itertools
 from codecs import BOM_UTF8
 
 from parso.python.token import PythonTokenTypes
-from parso._compatibility import py_version
 from parso.utils import split_lines
 
 
+# Maximum code point of Unicode 6.0: 0x10ffff (1,114,111)
+MAX_UNICODE = '\U0010ffff'
+
 STRING = PythonTokenTypes.STRING
 NAME = PythonTokenTypes.NAME
 NUMBER = PythonTokenTypes.NUMBER
@@ -47,12 +48,17 @@ BOM_UTF8_STRING = BOM_UTF8.decode('utf-8')
 
 _token_collection_cache = {}
 
-if py_version >= 30:
+if sys.version_info.major >= 3:
     # Python 3 has str.isidentifier() to check if a char is a valid identifier
     is_identifier = str.isidentifier
 else:
-    namechars = string.ascii_letters + '_'
-    is_identifier = lambda s: s in namechars
+    # Python 2 doesn't, but it's not that important anymore and if you tokenize
+    # Python 2 code with this, it's still ok. It's just that parsing Python 3
+    # code with this function is not 100% correct.
+    # This just means that Python 2 code matches a few identifiers too much,
+    # but that doesn't really matter.
+    def is_identifier(s):
+        return True
 
 
 def group(*choices, **kwargs):
@@ -78,7 +84,7 @@ def _all_string_prefixes(version_info, include_fstring=False, only_fstring=False
     #  and don't contain any permuations (include 'fr', but not
     #  'rf'). The various permutations will be generated.
     valid_string_prefixes = ['b', 'r', 'u']
-    if version_info >= (3, 0):
+    if version_info.major >= 3:
         valid_string_prefixes.append('br')
 
     result = set([''])
@@ -98,7 +104,7 @@ def _all_string_prefixes(version_info, include_fstring=False, only_fstring=False
             # create a list with upper and lower versions of each
             #  character
             result.update(different_case_versions(t))
-    if version_info <= (2, 7):
+    if version_info.major == 2:
         # In Python 2 the order cannot just be random.
         result.update(different_case_versions('ur'))
         result.update(different_case_versions('br'))
@@ -118,8 +124,10 @@ def _get_token_collection(version_info):
         return result
 
 
-fstring_string_single_line = _compile(r'(?:[^{}\r\n]+|\{\{|\}\})+')
+fstring_string_single_line = _compile(r'(?:\{\{|\}\}|\\(?:\r\n?|\n)|[^{}\r\n])+')
 fstring_string_multi_line = _compile(r'(?:[^{}]+|\{\{|\}\})+')
+fstring_format_spec_single_line = _compile(r'(?:\\(?:\r\n?|\n)|[^{}\r\n])+')
+fstring_format_spec_multi_line = _compile(r'[^{}]+')
 
 
 def _create_token_collection(version_info):
@@ -128,7 +136,16 @@ def _create_token_collection(version_info):
     Whitespace = r'[ \f\t]*'
     whitespace = _compile(Whitespace)
     Comment = r'#[^\r\n]*'
-    Name = r'\w+'
+    # Python 2 is pretty much not working properly anymore, we just ignore
+    # parsing unicode properly, which is fine, I guess.
+    if version_info[0] == 2:
+        Name = r'([A-Za-z_0-9]+)'
+    elif sys.version_info[0] == 2:
+        # Unfortunately the regex engine cannot deal with the regex below, so
+        # just use this one.
+        Name = r'(\w+)'
+    else:
+        Name = u'([A-Za-z_0-9\u0080-' + MAX_UNICODE + ']+)'
 
     if version_info >= (3, 6):
         Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
@@ -145,12 +162,14 @@ def _create_token_collection(version_info):
     else:
         Hexnumber = r'0[xX][0-9a-fA-F]+'
         Binnumber = r'0[bB][01]+'
-        if version_info >= (3, 0):
+        if version_info.major >= 3:
             Octnumber = r'0[oO][0-7]+'
         else:
             Octnumber = '0[oO]?[0-7]+'
         Decnumber = r'(?:0+|[1-9][0-9]*)'
         Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+        if version_info[0] < 3:
+            Intnumber += '[lL]?'
         Exponent = r'[eE][-+]?[0-9]+'
         Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
         Expfloat = r'[0-9]+' + Exponent
@@ -186,18 +205,22 @@ def _create_token_collection(version_info):
 
     Bracket = '[][(){}]'
 
-    special_args = [r'\r\n?', r'\n', r'[:;.,@]']
+    special_args = [r'\r\n?', r'\n', r'[;.,@]']
     if version_info >= (3, 0):
         special_args.insert(0, r'\.\.\.')
+    if version_info >= (3, 8):
+        special_args.insert(0, ":=?")
+    else:
+        special_args.insert(0, ":")
     Special = group(*special_args)
 
     Funny = group(Operator, Bracket, Special)
 
     # First (or only) line of ' or " string.
-    ContStr = group(StringPrefix + r"'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*" +
-                    group("'", r'\\(?:\r\n?|\n)'),
-                    StringPrefix + r'"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*' +
-                    group('"', r'\\(?:\r\n?|\n)'))
+    ContStr = group(StringPrefix + r"'[^\r\n'\\]*(?:\\.[^\r\n'\\]*)*"
+                    + group("'", r'\\(?:\r\n?|\n)'),
+                    StringPrefix + r'"[^\r\n"\\]*(?:\\.[^\r\n"\\]*)*'
+                    + group('"', r'\\(?:\r\n?|\n)'))
     pseudo_extra_pool = [Comment, Triple]
     all_quotes = '"', "'", '"""', "'''"
     if fstring_prefixes:
@@ -234,11 +257,14 @@ def _create_token_collection(version_info):
             fstring_pattern_map[t + quote] = quote
 
     ALWAYS_BREAK_TOKENS = (';', 'import', 'class', 'def', 'try', 'except',
-                           'finally', 'while', 'with', 'return')
+                           'finally', 'while', 'with', 'return', 'continue',
+                           'break', 'del', 'pass', 'global', 'assert')
+    if version_info >= (3, 5):
+        ALWAYS_BREAK_TOKENS += ('nonlocal', )
     pseudo_token_compiled = _compile(PseudoToken)
     return TokenCollection(
         pseudo_token_compiled, single_quoted, triple_quoted, endpats,
-        whitespace, fstring_pattern_map, ALWAYS_BREAK_TOKENS
+        whitespace, fstring_pattern_map, set(ALWAYS_BREAK_TOKENS)
     )
 
 
@@ -281,32 +307,45 @@ class FStringNode(object):
         return len(self.quote) == 3
 
     def is_in_expr(self):
-        return (self.parentheses_count - self.format_spec_count) > 0
+        return self.parentheses_count > self.format_spec_count
+
+    def is_in_format_spec(self):
+        return not self.is_in_expr() and self.format_spec_count
 
 
-def _close_fstring_if_necessary(fstring_stack, string, start_pos, additional_prefix):
+def _close_fstring_if_necessary(fstring_stack, string, line_nr, column, additional_prefix):
     for fstring_stack_index, node in enumerate(fstring_stack):
-        if string.startswith(node.quote):
+        lstripped_string = string.lstrip()
+        len_lstrip = len(string) - len(lstripped_string)
+        if lstripped_string.startswith(node.quote):
             token = PythonToken(
                 FSTRING_END,
                 node.quote,
-                start_pos,
-                prefix=additional_prefix,
+                (line_nr, column + len_lstrip),
+                prefix=additional_prefix+string[:len_lstrip],
             )
             additional_prefix = ''
             assert not node.previous_lines
             del fstring_stack[fstring_stack_index:]
-            return token, '', len(node.quote)
+            return token, '', len(node.quote) + len_lstrip
     return None, additional_prefix, 0
 
 
 def _find_fstring_string(endpats, fstring_stack, line, lnum, pos):
     tos = fstring_stack[-1]
     allow_multiline = tos.allow_multiline()
-    if allow_multiline:
-        match = fstring_string_multi_line.match(line, pos)
+    if tos.is_in_format_spec():
+        if allow_multiline:
+            regex = fstring_format_spec_multi_line
+        else:
+            regex = fstring_format_spec_single_line
     else:
-        match = fstring_string_single_line.match(line, pos)
+        if allow_multiline:
+            regex = fstring_string_multi_line
+        else:
+            regex = fstring_string_single_line
+
+    match = regex.match(line, pos)
     if match is None:
         return tos.previous_lines, pos
 
@@ -321,7 +360,9 @@ def _find_fstring_string(endpats, fstring_stack, line, lnum, pos):
 
     new_pos = pos
     new_pos += len(string)
-    if allow_multiline and (string.endswith('\n') or string.endswith('\r')):
+    # even if allow_multiline is False, we still need to check for trailing
+    # newlines, because a single-line f-string can contain line continuations
+    if string.endswith('\n') or string.endswith('\r'):
         tos.previous_lines += string
         string = ''
     else:
@@ -342,13 +383,14 @@ def _print_tokens(func):
     """
     def wrapper(*args, **kwargs):
         for token in func(*args, **kwargs):
+            print(token)  # This print is intentional for debugging!
             yield token
 
     return wrapper
 
 
 # @_print_tokens
-def tokenize_lines(lines, version_info, start_pos=(1, 0)):
+def tokenize_lines(lines, version_info, start_pos=(1, 0), indents=None, is_first_token=True):
     """
     A heavily modified Python standard library tokenizer.
 
@@ -359,17 +401,19 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
     def dedent_if_necessary(start):
         while start < indents[-1]:
             if start > indents[-2]:
-                yield PythonToken(ERROR_DEDENT, '', (lnum, 0), '')
+                yield PythonToken(ERROR_DEDENT, '', (lnum, start), '')
+                indents[-1] = start
                 break
-            yield PythonToken(DEDENT, '', spos, '')
             indents.pop()
+            yield PythonToken(DEDENT, '', spos, '')
 
     pseudo_token, single_quoted, triple_quoted, endpats, whitespace, \
         fstring_pattern_map, always_break_tokens, = \
         _get_token_collection(version_info)
     paren_level = 0  # count parentheses
-    indents = [0]
-    max = 0
+    if indents is None:
+        indents = [0]
+    max_ = 0
     numchars = '0123456789'
     contstr = ''
     contline = None
@@ -380,25 +424,24 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
     new_line = True
     prefix = ''  # Should never be required, but here for safety
     additional_prefix = ''
-    first = True
     lnum = start_pos[0] - 1
     fstring_stack = []
     for line in lines:  # loop over lines in stream
         lnum += 1
         pos = 0
-        max = len(line)
-        if first:
+        max_ = len(line)
+        if is_first_token:
             if line.startswith(BOM_UTF8_STRING):
                 additional_prefix = BOM_UTF8_STRING
                 line = line[1:]
-                max = len(line)
+                max_ = len(line)
 
             # Fake that the part before was already parsed.
             line = '^' * start_pos[1] + line
             pos = start_pos[1]
-            max += start_pos[1]
+            max_ += start_pos[1]
 
-            first = False
+            is_first_token = False
 
         if contstr:                                         # continued string
             endmatch = endprog.match(line)
@@ -414,13 +457,11 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 contline = contline + line
                 continue
 
-        while pos < max:
+        while pos < max_:
             if fstring_stack:
                 tos = fstring_stack[-1]
                 if not tos.is_in_expr():
                     string, pos = _find_fstring_string(endpats, fstring_stack, line, lnum, pos)
-                    if pos == max:
-                        break
                     if string:
                         yield PythonToken(
                             FSTRING_STRING, string,
@@ -431,12 +472,15 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                         )
                         tos.previous_lines = ''
                         continue
+                    if pos == max_:
+                        break
 
                 rest = line[pos:]
                 fstring_end_token, additional_prefix, quote_length = _close_fstring_if_necessary(
                     fstring_stack,
                     rest,
-                    (lnum, pos),
+                    lnum,
+                    pos,
                     additional_prefix,
                 )
                 pos += quote_length
@@ -444,10 +488,52 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                     yield fstring_end_token
                     continue
 
-            pseudomatch = pseudo_token.match(line, pos)
-            if not pseudomatch:                             # scan for tokens
+            # in an f-string, match until the end of the string
+            if fstring_stack:
+                string_line = line
+                for fstring_stack_node in fstring_stack:
+                    quote = fstring_stack_node.quote
+                    end_match = endpats[quote].match(line, pos)
+                    if end_match is not None:
+                        end_match_string = end_match.group(0)
+                        if len(end_match_string) - len(quote) + pos < len(string_line):
+                            string_line = line[:pos] + end_match_string[:-len(quote)]
+                pseudomatch = pseudo_token.match(string_line, pos)
+            else:
+                pseudomatch = pseudo_token.match(line, pos)
+
+            if pseudomatch:
+                prefix = additional_prefix + pseudomatch.group(1)
+                additional_prefix = ''
+                start, pos = pseudomatch.span(2)
+                spos = (lnum, start)
+                token = pseudomatch.group(2)
+                if token == '':
+                    assert prefix
+                    additional_prefix = prefix
+                    # This means that we have a line with whitespace/comments at
+                    # the end, which just results in an endmarker.
+                    break
+                initial = token[0]
+            else:
                 match = whitespace.match(line, pos)
-                if pos == 0:
+                initial = line[match.end()]
+                start = match.end()
+                spos = (lnum, start)
+
+            if new_line and initial not in '\r\n#' and (initial != '\\' or pseudomatch is None):
+                new_line = False
+                if paren_level == 0 and not fstring_stack:
+                    indent_start = start
+                    if indent_start > indents[-1]:
+                        yield PythonToken(INDENT, '', spos, '')
+                        indents.append(indent_start)
+                    for t in dedent_if_necessary(indent_start):
+                        yield t
+
+            if not pseudomatch:  # scan for tokens
+                match = whitespace.match(line, pos)
+                if new_line and paren_level == 0 and not fstring_stack:
                     for t in dedent_if_necessary(match.end()):
                         yield t
                 pos = match.end()
@@ -460,37 +546,23 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 pos += 1
                 continue
 
-            prefix = additional_prefix + pseudomatch.group(1)
-            additional_prefix = ''
-            start, pos = pseudomatch.span(2)
-            spos = (lnum, start)
-            token = pseudomatch.group(2)
-            if token == '':
-                assert prefix
-                additional_prefix = prefix
-                # This means that we have a line with whitespace/comments at
-                # the end, which just results in an endmarker.
-                break
-            initial = token[0]
-
-            if new_line and initial not in '\r\n\\#':
-                new_line = False
-                if paren_level == 0 and not fstring_stack:
-                    i = 0
-                    indent_start = start
-                    while line[i] == '\f':
-                        i += 1
-                        # TODO don't we need to change spos as well?
-                        indent_start -= 1
-                    if indent_start > indents[-1]:
-                        yield PythonToken(INDENT, '', spos, '')
-                        indents.append(indent_start)
-                    for t in dedent_if_necessary(indent_start):
-                        yield t
-
-            if (initial in numchars or                      # ordinary number
-                    (initial == '.' and token != '.' and token != '...')):
+            if (initial in numchars                      # ordinary number
+                    or (initial == '.' and token != '.' and token != '...')):
                 yield PythonToken(NUMBER, token, spos, prefix)
+            elif pseudomatch.group(3) is not None:            # ordinary name
+                if token in always_break_tokens and (fstring_stack or paren_level):
+                    fstring_stack[:] = []
+                    paren_level = 0
+                    # We only want to dedent if the token is on a new line.
+                    m = re.match(r'[ \f\t]*$', line[:start])
+                    if m is not None:
+                        for t in dedent_if_necessary(m.end()):
+                            yield t
+                if is_identifier(token):
+                    yield PythonToken(NAME, token, spos, prefix)
+                else:
+                    for t in _split_illegal_unicode_name(token, spos, prefix):
+                        yield t  # yield from Python 2
             elif initial in '\r\n':
                 if any(not f.allow_multiline() for f in fstring_stack):
                     # Would use fstring_stack.clear, but that's not available
@@ -504,7 +576,12 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 new_line = True
             elif initial == '#':  # Comments
                 assert not token.endswith("\n")
-                additional_prefix = prefix + token
+                if fstring_stack and fstring_stack[-1].is_in_expr():
+                    # `#` is not allowed in f-string expressions
+                    yield PythonToken(ERRORTOKEN, initial, spos, prefix)
+                    pos = start + 1
+                else:
+                    additional_prefix = prefix + token
             elif token in triple_quoted:
                 endprog = endpats[token]
                 endmatch = endprog.match(line, pos)
@@ -513,7 +590,7 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                     token = line[start:pos]
                     yield PythonToken(STRING, token, spos, prefix)
                 else:
-                    contstr_start = (lnum, start)           # multiple lines
+                    contstr_start = spos                    # multiple lines
                     contstr = line[start:]
                     contline = line
                     break
@@ -545,20 +622,6 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
             elif token in fstring_pattern_map:  # The start of an fstring.
                 fstring_stack.append(FStringNode(fstring_pattern_map[token]))
                 yield PythonToken(FSTRING_START, token, spos, prefix)
-            elif is_identifier(initial):                      # ordinary name
-                if token in always_break_tokens:
-                    fstring_stack[:] = []
-                    paren_level = 0
-                    # We only want to dedent if the token is on a new line.
-                    if re.match(r'[ \f\t]*$', line[:start]):
-                        while True:
-                            indent = indents.pop()
-                            if indent > start:
-                                yield PythonToken(DEDENT, '', spos, '')
-                            else:
-                                indents.append(indent)
-                                break
-                yield PythonToken(NAME, token, spos, prefix)
             elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n', '\\\r'):  # continued stmt
                 additional_prefix += prefix + line[start:]
                 break
@@ -574,9 +637,13 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                     else:
                         if paren_level:
                             paren_level -= 1
-                elif token == ':' and fstring_stack \
-                        and fstring_stack[-1].parentheses_count == 1:
+                elif token.startswith(':') and fstring_stack \
+                        and fstring_stack[-1].parentheses_count \
+                        - fstring_stack[-1].format_spec_count == 1:
+                    # `:` and `:=` both count
                     fstring_stack[-1].format_spec_count += 1
+                    token = ':'
+                    pos = start + 1
 
                 yield PythonToken(OP, token, spos, prefix)
 
@@ -585,14 +652,59 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
         if contstr.endswith('\n') or contstr.endswith('\r'):
             new_line = True
 
-    end_pos = lnum, max
+    if fstring_stack:
+        tos = fstring_stack[-1]
+        if tos.previous_lines:
+            yield PythonToken(
+                FSTRING_STRING, tos.previous_lines,
+                tos.last_string_start_pos,
+                # Never has a prefix because it can start anywhere and
+                # include whitespace.
+                prefix=''
+            )
+
+    end_pos = lnum, max_
     # As the last position we just take the maximally possible position. We
     # remove -1 for the last new line.
     for indent in indents[1:]:
+        indents.pop()
         yield PythonToken(DEDENT, '', end_pos, '')
     yield PythonToken(ENDMARKER, '', end_pos, additional_prefix)
 
 
+def _split_illegal_unicode_name(token, start_pos, prefix):
+    def create_token():
+        return PythonToken(ERRORTOKEN if is_illegal else NAME, found, pos, prefix)
+
+    found = ''
+    is_illegal = False
+    pos = start_pos
+    for i, char in enumerate(token):
+        if is_illegal:
+            if is_identifier(char):
+                yield create_token()
+                found = char
+                is_illegal = False
+                prefix = ''
+                pos = start_pos[0], start_pos[1] + i
+            else:
+                found += char
+        else:
+            new_found = found + char
+            if is_identifier(new_found):
+                found = new_found
+            else:
+                if found:
+                    yield create_token()
+                    prefix = ''
+                    pos = start_pos[0], start_pos[1] + i
+                found = char
+                is_illegal = True
+
+    if found:
+        yield create_token()
+
+
 if __name__ == "__main__":
     if len(sys.argv) >= 2:
         path = sys.argv[1]

parso/python/tokenize.pyi

@@ -0,0 +1,24 @@
+from typing import Generator, Iterable, NamedTuple, Tuple
+
+from parso.python.token import TokenType
+from parso.utils import PythonVersionInfo
+
+class Token(NamedTuple):
+    type: TokenType
+    string: str
+    start_pos: Tuple[int, int]
+    prefix: str
+    @property
+    def end_pos(self) -> Tuple[int, int]: ...
+
+class PythonToken(Token):
+    def __repr__(self) -> str: ...
+
+def tokenize(
+    code: str, version_info: PythonVersionInfo, start_pos: Tuple[int, int] = (1, 0)
+) -> Generator[PythonToken, None, None]: ...
+def tokenize_lines(
+    lines: Iterable[str],
+    version_info: PythonVersionInfo,
+    start_pos: Tuple[int, int] = (1, 0),
+) -> Generator[PythonToken, None, None]: ...

parso/python/tree.py

@@ -43,6 +43,10 @@ Parser Tree Classes
 """
 
 import re
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
 
 from parso._compatibility import utf8_repr, unicode
 from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf, \
@@ -53,10 +57,14 @@ from parso.utils import split_lines
 _FLOW_CONTAINERS = set(['if_stmt', 'while_stmt', 'for_stmt', 'try_stmt',
                         'with_stmt', 'async_stmt', 'suite'])
 _RETURN_STMT_CONTAINERS = set(['suite', 'simple_stmt']) | _FLOW_CONTAINERS
-_FUNC_CONTAINERS = set(['suite', 'simple_stmt', 'decorated']) | _FLOW_CONTAINERS
+
+_FUNC_CONTAINERS = set(
+    ['suite', 'simple_stmt', 'decorated', 'async_funcdef']
+) | _FLOW_CONTAINERS
+
 _GET_DEFINITION_TYPES = set([
-    'expr_stmt', 'comp_for', 'with_stmt', 'for_stmt', 'import_name',
-    'import_from', 'param'
+    'expr_stmt', 'sync_comp_for', 'with_stmt', 'for_stmt', 'import_name',
+    'import_from', 'param', 'del_stmt',
 ])
 _IMPORTS = set(['import_name', 'import_from'])
 
@@ -91,7 +99,7 @@ class DocstringMixin(object):
 
 class PythonMixin(object):
     """
-    Some Python specific utitilies.
+    Some Python specific utilities.
     """
     __slots__ = ()
 
@@ -196,25 +204,22 @@ class Name(_LeafWithoutNewlines):
         return "<%s: %s@%s,%s>" % (type(self).__name__, self.value,
                                    self.line, self.column)
 
-    def is_definition(self):
+    def is_definition(self, include_setitem=False):
         """
         Returns True if the name is being defined.
         """
-        return self.get_definition() is not None
+        return self.get_definition(include_setitem=include_setitem) is not None
 
-    def get_definition(self, import_name_always=False):
+    def get_definition(self, import_name_always=False, include_setitem=False):
         """
-        Returns None if there's on definition for a name.
+        Returns None if there's no definition for a name.
 
-        :param import_name_alway: Specifies if an import name is always a
+        :param import_name_always: Specifies if an import name is always a
             definition. Normally foo in `from foo import bar` is not a
             definition.
         """
         node = self.parent
         type_ = node.type
-        if type_ in ('power', 'atom_expr'):
-            # In `self.x = 3` self is not a definition, but x is.
-            return None
 
         if type_ in ('funcdef', 'classdef'):
             if self == node.name:
@@ -232,8 +237,10 @@ class Name(_LeafWithoutNewlines):
         while node is not None:
             if node.type == 'suite':
                 return None
+            if node.type == 'namedexpr_test':
+                return node.children[0]
             if node.type in _GET_DEFINITION_TYPES:
-                if self in node.get_defined_names():
+                if self in node.get_defined_names(include_setitem):
                     return node
                 if import_name_always and node.type in _IMPORTS:
                     return node
@@ -442,7 +449,7 @@ class Module(Scope):
                         recurse(child)
 
             recurse(self)
-            self._used_names = dct
+            self._used_names = UsedNamesMapping(dct)
         return self._used_names
 
 
@@ -466,6 +473,9 @@ class ClassOrFunc(Scope):
         :rtype: list of :class:`Decorator`
         """
         decorated = self.parent
+        if decorated.type == 'async_funcdef':
+            decorated = decorated.parent
+
         if decorated.type == 'decorated':
             if decorated.children[0].type == 'decorators':
                 return decorated.children[0].children
@@ -545,7 +555,8 @@ def _create_params(parent, argslist_list):
                     if param_children[0] == '*' \
                             and (len(param_children) == 1
                                  or param_children[1] == ',') \
-                            or check_python2_nested_param(param_children[0]):
+                            or check_python2_nested_param(param_children[0]) \
+                            or param_children[0] == '/':
                         for p in param_children:
                             p.parent = parent
                         new_children += param_children
@@ -767,8 +778,8 @@ class ForStmt(Flow):
         """
         return self.children[3]
 
-    def get_defined_names(self):
-        return _defined_names(self.children[1])
+    def get_defined_names(self, include_setitem=False):
+        return _defined_names(self.children[1], include_setitem)
 
 
 class TryStmt(Flow):
@@ -791,7 +802,7 @@ class WithStmt(Flow):
     type = 'with_stmt'
     __slots__ = ()
 
-    def get_defined_names(self):
+    def get_defined_names(self, include_setitem=False):
         """
         Returns the a list of `Name` that the with statement defines. The
         defined names are set after `as`.
@@ -800,7 +811,7 @@ class WithStmt(Flow):
         for with_item in self.children[1:-2:2]:
             # Check with items for 'as' names.
             if with_item.type == 'with_item':
-                names += _defined_names(with_item.children[2])
+                names += _defined_names(with_item.children[2], include_setitem)
         return names
 
     def get_test_node_from_name(self, name):
@@ -841,7 +852,7 @@ class ImportFrom(Import):
     type = 'import_from'
     __slots__ = ()
 
-    def get_defined_names(self):
+    def get_defined_names(self, include_setitem=False):
         """
         Returns the a list of `Name` that the import defines. The
         defined names are set after `import` or in case an alias - `as` - is
@@ -912,7 +923,7 @@ class ImportName(Import):
     type = 'import_name'
     __slots__ = ()
 
-    def get_defined_names(self):
+    def get_defined_names(self, include_setitem=False):
         """
         Returns the a list of `Name` that the import defines. The defined names
         is always the first name after `import` or in case an alias - `as` - is
@@ -969,7 +980,7 @@ class ImportName(Import):
 class KeywordStatement(PythonBaseNode):
     """
     For the following statements: `assert`, `del`, `global`, `nonlocal`,
-    `raise`, `return`, `yield`, `return`, `yield`.
+    `raise`, `return`, `yield`.
 
     `pass`, `continue` and `break` are not in there, because they are just
     simple keywords and the parser reduces it to a keyword.
@@ -988,6 +999,14 @@ class KeywordStatement(PythonBaseNode):
     def keyword(self):
         return self.children[0].value
 
+    def get_defined_names(self, include_setitem=False):
+        keyword = self.keyword
+        if keyword == 'del':
+            return _defined_names(self.children[1], include_setitem)
+        if keyword in ('global', 'nonlocal'):
+            return self.children[1::2]
+        return []
+
 
 class AssertStmt(KeywordStatement):
     __slots__ = ()
@@ -1013,7 +1032,7 @@ class YieldExpr(PythonBaseNode):
     __slots__ = ()
 
 
-def _defined_names(current):
+def _defined_names(current, include_setitem):
     """
     A helper function to find the defined names in statements, for loops and
     list comprehensions.
@@ -1021,14 +1040,22 @@ def _defined_names(current):
     names = []
     if current.type in ('testlist_star_expr', 'testlist_comp', 'exprlist', 'testlist'):
         for child in current.children[::2]:
-            names += _defined_names(child)
+            names += _defined_names(child, include_setitem)
     elif current.type in ('atom', 'star_expr'):
-        names += _defined_names(current.children[1])
+        names += _defined_names(current.children[1], include_setitem)
     elif current.type in ('power', 'atom_expr'):
         if current.children[-2] != '**':  # Just if there's no operation
             trailer = current.children[-1]
             if trailer.children[0] == '.':
                 names.append(trailer.children[1])
+            elif trailer.children[0] == '[' and include_setitem:
+                for node in current.children[-2::-1]:
+                    if node.type == 'trailer':
+                        names.append(node.children[1])
+                        break
+                    if node.type == 'name':
+                        names.append(node)
+                        break
     else:
         names.append(current)
     return names
@@ -1038,23 +1065,29 @@ class ExprStmt(PythonBaseNode, DocstringMixin):
     type = 'expr_stmt'
     __slots__ = ()
 
-    def get_defined_names(self):
+    def get_defined_names(self, include_setitem=False):
         """
         Returns a list of `Name` defined before the `=` sign.
         """
         names = []
         if self.children[1].type == 'annassign':
-            names = _defined_names(self.children[0])
+            names = _defined_names(self.children[0], include_setitem)
         return [
             name
             for i in range(0, len(self.children) - 2, 2)
             if '=' in self.children[i + 1].value
-            for name in _defined_names(self.children[i])
+            for name in _defined_names(self.children[i], include_setitem)
         ] + names
 
     def get_rhs(self):
         """Returns the right-hand-side of the equals."""
-        return self.children[-1]
+        node = self.children[-1]
+        if node.type == 'annassign':
+            if len(node.children) == 4:
+                node = node.children[3]
+            else:
+                node = node.children[1]
+        return node
 
     def yield_operators(self):
         """
@@ -1142,7 +1175,7 @@ class Param(PythonBaseNode):
         else:
             return self._tfpdef()
 
-    def get_defined_names(self):
+    def get_defined_names(self, include_setitem=False):
         return [self.name]
 
     @property
@@ -1158,6 +1191,13 @@ class Param(PythonBaseNode):
                 index -= 2
         except ValueError:
             pass
+        try:
+            keyword_only_index = self.parent.children.index('/')
+            if index > keyword_only_index:
+                # Skip the ` /, `
+                index -= 2
+        except ValueError:
+            pass
         return index - 1
 
     def get_parent_function(self):
@@ -1189,13 +1229,42 @@ class Param(PythonBaseNode):
         return '<%s: %s>' % (type(self).__name__, str(self._tfpdef()) + default)
 
 
-class CompFor(PythonBaseNode):
-    type = 'comp_for'
+class SyncCompFor(PythonBaseNode):
+    type = 'sync_comp_for'
     __slots__ = ()
 
-    def get_defined_names(self):
+    def get_defined_names(self, include_setitem=False):
         """
         Returns the a list of `Name` that the comprehension defines.
         """
         # allow async for
-        return _defined_names(self.children[self.children.index('for') + 1])
+        return _defined_names(self.children[1], include_setitem)
+
+
+# This is simply here so an older Jedi version can work with this new parso
+# version. Can be deleted in the next release.
+CompFor = SyncCompFor
+
+
+class UsedNamesMapping(Mapping):
+    """
+    This class exists for the sole purpose of creating an immutable dict.
+    """
+    def __init__(self, dct):
+        self._dict = dct
+
+    def __getitem__(self, key):
+        return self._dict[key]
+
+    def __len__(self):
+        return len(self._dict)
+
+    def __iter__(self):
+        return iter(self._dict)
+
+    def __hash__(self):
+        return id(self)
+
+    def __eq__(self, other):
+        # Comparing these dicts does not make sense.
+        return self is other

parso/tree.py

@@ -1,6 +1,7 @@
+import sys
 from abc import abstractmethod, abstractproperty
 
-from parso._compatibility import utf8_repr, encoding, py_version
+from parso._compatibility import utf8_repr, encoding
 from parso.utils import split_lines
 
 
@@ -44,8 +45,12 @@ class NodeOrLeaf(object):
         Returns the node immediately following this node in this parent's
         children list. If this node does not have a next sibling, it is None
         """
+        parent = self.parent
+        if parent is None:
+            return None
+
         # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
+        for i, child in enumerate(parent.children):
             if child is self:
                 try:
                     return self.parent.children[i + 1]
@@ -58,8 +63,12 @@ class NodeOrLeaf(object):
         children list. If this node does not have a previous sibling, it is
         None.
         """
+        parent = self.parent
+        if parent is None:
+            return None
+
         # Can't use index(); we need to test by identity
-        for i, child in enumerate(self.parent.children):
+        for i, child in enumerate(parent.children):
             if child is self:
                 if i == 0:
                     return None
@@ -70,6 +79,9 @@ class NodeOrLeaf(object):
         Returns the previous leaf in the parser tree.
         Returns `None` if this is the first element in the parser tree.
         """
+        if self.parent is None:
+            return None
+
         node = self
         while True:
             c = node.parent.children
@@ -93,6 +105,9 @@ class NodeOrLeaf(object):
         Returns the next leaf in the parser tree.
         Returns None if this is the last element in the parser tree.
         """
+        if self.parent is None:
+            return None
+
         node = self
         while True:
             c = node.parent.children
@@ -153,7 +168,7 @@ class NodeOrLeaf(object):
     @abstractmethod
     def get_code(self, include_prefix=True):
         """
-        Returns the code that was input the input for the parser for this node.
+        Returns the code that was the input for the parser for this node.
 
         :param include_prefix: Removes the prefix (whitespace and comments) of
             e.g. a statement.
@@ -321,7 +336,7 @@ class BaseNode(NodeOrLeaf):
     @utf8_repr
     def __repr__(self):
         code = self.get_code().replace('\n', ' ').replace('\r', ' ').strip()
-        if not py_version >= 30:
+        if not sys.version_info.major >= 3:
             code = code.encode(encoding, 'replace')
         return "<%s: %s@%s,%s>" % \
             (type(self).__name__, code, self.start_pos[0], self.start_pos[1])

parso/utils.py

@@ -2,8 +2,9 @@ from collections import namedtuple
 import re
 import sys
 from ast import literal_eval
+from functools import total_ordering
 
-from parso._compatibility import unicode, total_ordering
+from parso._compatibility import unicode
 
 # The following is a list in Python that are line breaks in str.splitlines, but
 # not in Python. In Python only \r (Carriage Return, 0xD) and \n (Line Feed,
@@ -104,8 +105,17 @@ def python_bytes_to_unicode(source, encoding='utf-8', errors='strict'):
     if not isinstance(encoding, unicode):
         encoding = unicode(encoding, 'utf-8', 'replace')
 
-    # Cast to unicode
-    return unicode(source, encoding, errors)
+    try:
+        # Cast to unicode
+        return unicode(source, encoding, errors)
+    except LookupError:
+        if errors == 'replace':
+            # This is a weird case that can happen if the given encoding is not
+            # a valid encoding. This usually shouldn't happen with provided
+            # encodings, but can happen if somebody uses encoding declarations
+            # like `# coding: foo-8`.
+            return unicode(source, 'utf-8', errors)
+        raise
 
 
 def version_info():
@@ -119,10 +129,10 @@ def version_info():
 
 
 def _parse_version(version):
-    match = re.match(r'(\d+)(?:\.(\d)(?:\.\d+)?)?$', version)
+    match = re.match(r'(\d+)(?:\.(\d{1,2})(?:\.\d+)?)?((a|b|rc)\d)?$', version)
     if match is None:
         raise ValueError('The given version is not in the right format. '
-                         'Use something like "3.2" or "3".')
+                         'Use something like "3.8" or "3".')
 
     major = int(match.group(1))
     minor = match.group(2)
@@ -163,13 +173,13 @@ class PythonVersionInfo(namedtuple('Version', 'major, minor')):
 
 def parse_version_string(version=None):
     """
-    Checks for a valid version number (e.g. `3.2` or `2.7.1` or `3`) and
+    Checks for a valid version number (e.g. `3.8` or `2.7.1` or `3`) and
     returns a corresponding version info that is always two characters long in
     decimal.
     """
     if version is None:
         version = '%s.%s' % sys.version_info[:2]
     if not isinstance(version, (unicode, str)):
-        raise TypeError("version must be a string like 3.2.")
+        raise TypeError('version must be a string like "3.8"')
 
     return _parse_version(version)

parso/utils.pyi

@@ -0,0 +1,29 @@
+from typing import NamedTuple, Optional, Sequence, Union
+
+class Version(NamedTuple):
+    major: int
+    minor: int
+    micro: int
+
+def split_lines(string: str, keepends: bool = ...) -> Sequence[str]: ...
+def python_bytes_to_unicode(
+    source: Union[str, bytes], encoding: str = ..., errors: str = ...
+) -> str: ...
+def version_info() -> Version:
+    """
+    Returns a namedtuple of parso's version, similar to Python's
+    ``sys.version_info``.
+    """
+    ...
+
+class PythonVersionInfo(NamedTuple):
+    major: int
+    minor: int
+
+def parse_version_string(version: Optional[str]) -> PythonVersionInfo:
+    """
+    Checks for a valid version number (e.g. `3.2` or `2.7.1` or `3`) and
+    returns a corresponding version info that is always two characters long in
+    decimal.
+    """
+    ...

pytest.ini

@@ -1,6 +1,8 @@
 [pytest]
 addopts = --doctest-modules
 
+testpaths = parso test
+
 # Ignore broken files inblackbox test directories
 norecursedirs = .* docs scripts normalizer_issue_files build
 

setup.cfg

@@ -1,2 +1,12 @@
 [bdist_wheel]
 universal=1
+
+[flake8]
+max-line-length = 100
+ignore =
+  # do not use bare 'except'
+  E722,
+  # don't know why this was ever even an option, 1+1 should be possible.
+  E226,
+  # line break before binary operator
+  W503,

setup.py

@@ -25,8 +25,9 @@ setup(name='parso',
       keywords='python parser parsing',
       long_description=readme,
       packages=find_packages(exclude=['test']),
-      package_data={'parso': ['python/grammar*.txt']},
+      package_data={'parso': ['python/grammar*.txt', 'py.typed', '*.pyi', '**/*.pyi']},
       platforms=['any'],
+      python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
       classifiers=[
           'Development Status :: 4 - Beta',
           'Environment :: Plugins',
@@ -34,10 +35,8 @@ setup(name='parso',
           'License :: OSI Approved :: MIT License',
           'Operating System :: OS Independent',
           'Programming Language :: Python :: 2',
-          'Programming Language :: Python :: 2.6',
           'Programming Language :: Python :: 2.7',
           'Programming Language :: Python :: 3',
-          'Programming Language :: Python :: 3.3',
           'Programming Language :: Python :: 3.4',
           'Programming Language :: Python :: 3.5',
           'Programming Language :: Python :: 3.6',
@@ -45,6 +44,7 @@ setup(name='parso',
           'Topic :: Software Development :: Libraries :: Python Modules',
           'Topic :: Text Editors :: Integrated Development Environments (IDE)',
           'Topic :: Utilities',
+          'Typing :: Typed',
       ],
       extras_require={
           'testing': [

test/failing_examples.py

@@ -19,14 +19,6 @@ def build_nested(code, depth, base='def f():\n'):
 FAILING_EXAMPLES = [
     '1 +',
     '?',
-    # Python/compile.c
-    dedent('''\
-        for a in [1]:
-            try:
-                pass
-            finally:
-                continue
-        '''), # 'continue' not supported inside 'finally' clause"
     'continue',
     'break',
     'return',
@@ -60,9 +52,37 @@ FAILING_EXAMPLES = [
     'f(x=2, y)',
     'f(**x, *y)',
     'f(**x, y=3, z)',
+    # augassign
     'a, b += 3',
     '(a, b) += 3',
     '[a, b] += 3',
+    'f() += 1',
+    'lambda x:None+=1',
+    '{} += 1',
+    '{a:b} += 1',
+    '{1} += 1',
+    '{*x} += 1',
+    '(x,) += 1',
+    '(x, y if a else q) += 1',
+    '[] += 1',
+    '[1,2] += 1',
+    '[] += 1',
+    'None += 1',
+    '... += 1',
+    'a > 1 += 1',
+    '"test" += 1',
+    '1 += 1',
+    '1.0 += 1',
+    '(yield) += 1',
+    '(yield from x) += 1',
+    '(x if x else y) += 1',
+    'a() += 1',
+    'a + b += 1',
+    '+a += 1',
+    'a and b += 1',
+    '*a += 1',
+    'a, b += 1',
+    'f"xxx" += 1',
     # All assignment tests
     'lambda a: 1 = 1',
     '[x for x in y] = 1',
@@ -154,7 +174,7 @@ FAILING_EXAMPLES = [
     # Now nested parsing
     "f'{continue}'",
     "f'{1;1}'",
-    "f'{a=3}'",
+    "f'{a;}'",
     "f'{b\"\" \"\"}'",
 ]
 
@@ -259,10 +279,6 @@ GLOBAL_NONLOCAL_ERROR = [
 
 if sys.version_info >= (3, 6):
     FAILING_EXAMPLES += GLOBAL_NONLOCAL_ERROR
-    FAILING_EXAMPLES += [
-        # Raises multiple errors in previous versions.
-        'async def foo():\n def nofoo():[x async for x in []]',
-    ]
 if sys.version_info >= (3, 5):
     FAILING_EXAMPLES += [
         # Raises different errors so just ignore them for now.
@@ -285,11 +301,19 @@ if sys.version_info >= (3,):
         'b"ä"',
         # combining strings and unicode is allowed in Python 2.
         '"s" b""',
+        '"s" b"" ""',
+        'b"" "" b"" ""',
     ]
-if sys.version_info >= (2, 7):
-    # This is something that raises a different error in 2.6 than in the other
-    # versions. Just skip it for 2.6.
-    FAILING_EXAMPLES.append('[a, 1] += 3')
+if sys.version_info >= (3, 6):
+    FAILING_EXAMPLES += [
+        # Same as above, but for f-strings.
+        'f"s" b""',
+        'b"s" f""',
+
+        # f-string expression part cannot include a backslash
+        r'''f"{'\n'}"''',
+    ]
+FAILING_EXAMPLES.append('[a, 1] += 3')
 
 if sys.version_info[:2] == (3, 5):
     # yields are not allowed in 3.5 async functions. Therefore test them
@@ -311,3 +335,63 @@ if sys.version_info[:2] <= (3, 4):
         'a = *[1], 2',
         '(*[1], 2)',
     ]
+
+if sys.version_info[:2] >= (3, 7):
+    # This is somehow ok in previous versions.
+    FAILING_EXAMPLES += [
+        'class X(base for base in bases): pass',
+    ]
+
+if sys.version_info[:2] < (3, 8):
+    FAILING_EXAMPLES += [
+        # Python/compile.c
+        dedent('''\
+            for a in [1]:
+                try:
+                    pass
+                finally:
+                    continue
+            '''),  # 'continue' not supported inside 'finally' clause"
+    ]
+
+if sys.version_info[:2] >= (3, 8):
+    # assignment expressions from issue#89
+    FAILING_EXAMPLES += [
+        # Case 2
+        '(lambda: x := 1)',
+        '((lambda: x) := 1)',
+        # Case 3
+        '(a[i] := x)',
+        '((a[i]) := x)',
+        '(a(i) := x)',
+        # Case 4
+        '(a.b := c)',
+        '[(i.i:= 0) for ((i), j) in range(5)]',
+        # Case 5
+        '[i:= 0 for i, j in range(5)]',
+        '[(i:= 0) for ((i), j) in range(5)]',
+        '[(i:= 0) for ((i), j), in range(5)]',
+        '[(i:= 0) for ((i), j.i), in range(5)]',
+        '[[(i:= i) for j in range(5)] for i in range(5)]',
+        '[i for i, j in range(5) if True or (i:= 1)]',
+        '[False and (i:= 0) for i, j in range(5)]',
+        # Case 6
+        '[i+1 for i in (i:= range(5))]',
+        '[i+1 for i in (j:= range(5))]',
+        '[i+1 for i in (lambda: (j:= range(5)))()]',
+        # Case 7
+        'class Example:\n [(j := i) for i in range(5)]',
+        # Not in that issue
+        '(await a := x)',
+        '((await a) := x)',
+        # new discoveries
+        '((a, b) := (1, 2))',
+        '([a, b] := [1, 2])',
+        '({a, b} := {1, 2})',
+        '({a: b} := {1: 2})',
+        '(a + b := 1)',
+        '(True := 1)',
+        '(False := 1)',
+        '(None := 1)',
+        '(__debug__ := 1)',
+    ]

test/fuzz_diff_parser.py

@@ -50,6 +50,11 @@ def find_python_files_in_tree(file_path):
         yield file_path
         return
     for root, dirnames, filenames in os.walk(file_path):
+        if 'chardet' in root:
+            # Stuff like chardet/langcyrillicmodel.py is just very slow to
+            # parse and machine generated, so ignore those.
+            continue
+
         for name in filenames:
             if name.endswith('.py'):
                 yield os.path.join(root, name)
@@ -102,9 +107,17 @@ class LineCopy:
 
 class FileModification:
     @classmethod
-    def generate(cls, code_lines, change_count):
+    def generate(cls, code_lines, change_count, previous_file_modification=None):
+        if previous_file_modification is not None and random.random() > 0.5:
+            # We want to keep the previous modifications in some cases to make
+            # more complex parser issues visible.
+            code_lines = previous_file_modification.apply(code_lines)
+            added_modifications = previous_file_modification.modification_list
+        else:
+            added_modifications = []
         return cls(
-            list(cls._generate_line_modifications(code_lines, change_count)),
+            added_modifications
+            + list(cls._generate_line_modifications(code_lines, change_count)),
             # work with changed trees more than with normal ones.
             check_original=random.random() > 0.8,
         )
@@ -158,18 +171,18 @@ class FileModification:
             yield l
 
     def __init__(self, modification_list, check_original):
-        self._modification_list = modification_list
+        self.modification_list = modification_list
         self._check_original = check_original
 
-    def _apply(self, code_lines):
+    def apply(self, code_lines):
         changed_lines = list(code_lines)
-        for modification in self._modification_list:
+        for modification in self.modification_list:
             modification.apply(changed_lines)
         return changed_lines
 
     def run(self, grammar, code_lines, print_code):
         code = ''.join(code_lines)
-        modified_lines = self._apply(code_lines)
+        modified_lines = self.apply(code_lines)
         modified_code = ''.join(modified_lines)
 
         if print_code:
@@ -197,15 +210,12 @@ class FileModification:
 class FileTests:
     def __init__(self, file_path, test_count, change_count):
         self._path = file_path
-        with open(file_path) as f:
+        with open(file_path, errors='replace') as f:
             code = f.read()
         self._code_lines = split_lines(code, keepends=True)
         self._test_count = test_count
         self._code_lines = self._code_lines
         self._change_count = change_count
-
-        with open(file_path) as f:
-            code = f.read()
         self._file_modifications = []
 
     def _run(self, grammar, file_modifications, debugger, print_code=False):
@@ -231,8 +241,12 @@ class FileTests:
 
     def run(self, grammar, debugger):
         def iterate():
+            fm = None
             for _ in range(self._test_count):
-                fm = FileModification.generate(self._code_lines, self._change_count)
+                fm = FileModification.generate(
+                    self._code_lines, self._change_count,
+                    previous_file_modification=fm
+                )
                 self._file_modifications.append(fm)
                 yield fm
 

test/normalizer_issue_files/allowed_syntax.py

@@ -12,13 +12,6 @@ from .__future__ import absolute_import
 ''r''u''
 b'' BR''
 
-for x in [1]:
-    try:
-        continue  # Only the other continue and pass is an error.
-    finally:
-        #: E901
-        continue
-
 
 for x in [1]:
     break

test/test_cache.py

@@ -2,25 +2,38 @@
 Test all things related to the ``jedi.cache`` module.
 """
 
-from os import unlink
+import os
+import os.path
 
 import pytest
+import time
 
-from parso.cache import _NodeCacheItem, save_module, load_module, \
-    _get_hashed_path, parser_cache, _load_from_file_system, _save_to_file_system
+from parso.cache import (_CACHED_FILE_MAXIMUM_SURVIVAL, _VERSION_TAG,
+                         _get_cache_clear_lock, _get_hashed_path,
+                         _load_from_file_system, _NodeCacheItem,
+                         _remove_cache_and_update_lock, _save_to_file_system,
+                         load_module, parser_cache, try_to_save_module)
+from parso._compatibility import is_pypy, PermissionError
 from parso import load_grammar
 from parso import cache
+from parso import file_io
+from parso import parse
+
+skip_pypy = pytest.mark.skipif(
+    is_pypy,
+    reason="pickling in pypy is slow, since we don't pickle,"
+           "we never go into path of auto-collecting garbage"
+)
 
 
 @pytest.fixture()
-def isolated_jedi_cache(monkeypatch, tmpdir):
-    """
-    Set `jedi.settings.cache_directory` to a temporary directory during test.
-
-    Same as `clean_jedi_cache`, but create the temporary directory for
-    each test case (scope='function').
-    """
-    monkeypatch.setattr(cache, '_default_cache_path', str(tmpdir))
+def isolated_parso_cache(monkeypatch, tmpdir):
+    """Set `parso.cache._default_cache_path` to a temporary directory
+    during the test. """
+    cache_path = str(os.path.join(str(tmpdir), "__parso_cache"))
+    monkeypatch.setattr(cache, '_default_cache_path', cache_path)
+    monkeypatch.setattr(cache, '_get_default_cache_path', lambda *args, **kwargs: cache_path)
+    return cache_path
 
 
 def test_modulepickling_change_cache_dir(tmpdir):
@@ -54,7 +67,7 @@ def load_stored_item(hashed_grammar, path, item, cache_path):
     return item
 
 
-@pytest.mark.usefixtures("isolated_jedi_cache")
+@pytest.mark.usefixtures("isolated_parso_cache")
 def test_modulepickling_simulate_deleted_cache(tmpdir):
     """
     Tests loading from a cache file after it is deleted.
@@ -76,12 +89,103 @@ def test_modulepickling_simulate_deleted_cache(tmpdir):
     path = tmpdir.dirname + '/some_path'
     with open(path, 'w'):
         pass
+    io = file_io.FileIO(path)
 
-    save_module(grammar._hashed, path, module, [])
-    assert load_module(grammar._hashed, path) == module
+    try_to_save_module(grammar._hashed, io, module, lines=[])
+    assert load_module(grammar._hashed, io) == module
 
-    unlink(_get_hashed_path(grammar._hashed, path))
+    os.unlink(_get_hashed_path(grammar._hashed, path))
     parser_cache.clear()
 
-    cached2 = load_module(grammar._hashed, path)
+    cached2 = load_module(grammar._hashed, io)
     assert cached2 is None
+
+
+def test_cache_limit():
+    def cache_size():
+        return sum(len(v) for v in parser_cache.values())
+
+    try:
+        parser_cache.clear()
+        future_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() + 10e6)
+        old_node_cache_item = _NodeCacheItem('bla', [], change_time=time.time() - 10e4)
+        parser_cache['some_hash_old'] = {
+            '/path/%s' % i: old_node_cache_item for i in range(300)
+        }
+        parser_cache['some_hash_new'] = {
+            '/path/%s' % i: future_node_cache_item for i in range(300)
+        }
+        assert cache_size() == 600
+        parse('somecode', cache=True, path='/path/somepath')
+        assert cache_size() == 301
+    finally:
+        parser_cache.clear()
+
+
+class _FixedTimeFileIO(file_io.KnownContentFileIO):
+    def __init__(self, path, content, last_modified):
+        super(_FixedTimeFileIO, self).__init__(path, content)
+        self._last_modified = last_modified
+
+    def get_last_modified(self):
+        return self._last_modified
+
+
+@pytest.mark.parametrize('diff_cache', [False, True])
+@pytest.mark.parametrize('use_file_io', [False, True])
+def test_cache_last_used_update(diff_cache, use_file_io):
+    p = '/path/last-used'
+    parser_cache.clear()  # Clear, because then it's easier to find stuff.
+    parse('somecode', cache=True, path=p)
+    node_cache_item = next(iter(parser_cache.values()))[p]
+    now = time.time()
+    assert node_cache_item.last_used < now
+
+    if use_file_io:
+        f = _FixedTimeFileIO(p, 'code', node_cache_item.last_used - 10)
+        parse(file_io=f, cache=True, diff_cache=diff_cache)
+    else:
+        parse('somecode2', cache=True, path=p, diff_cache=diff_cache)
+
+    node_cache_item = next(iter(parser_cache.values()))[p]
+    assert now < node_cache_item.last_used < time.time()
+
+
+@skip_pypy
+def test_inactive_cache(tmpdir, isolated_parso_cache):
+    parser_cache.clear()
+    test_subjects = "abcdef"
+    for path in test_subjects:
+        parse('somecode', cache=True, path=os.path.join(str(tmpdir), path))
+    raw_cache_path = os.path.join(isolated_parso_cache, _VERSION_TAG)
+    assert os.path.exists(raw_cache_path)
+    paths = os.listdir(raw_cache_path)
+    a_while_ago = time.time() - _CACHED_FILE_MAXIMUM_SURVIVAL
+    old_paths = set()
+    for path in paths[:len(test_subjects) // 2]:  # make certain number of paths old
+        os.utime(os.path.join(raw_cache_path, path), (a_while_ago, a_while_ago))
+        old_paths.add(path)
+    # nothing should be cleared while the lock is on
+    assert os.path.exists(_get_cache_clear_lock().path)
+    _remove_cache_and_update_lock()  # it shouldn't clear anything
+    assert len(os.listdir(raw_cache_path)) == len(test_subjects)
+    assert old_paths.issubset(os.listdir(raw_cache_path))
+
+    os.utime(_get_cache_clear_lock().path, (a_while_ago, a_while_ago))
+    _remove_cache_and_update_lock()
+    assert len(os.listdir(raw_cache_path)) == len(test_subjects) // 2
+    assert not old_paths.intersection(os.listdir(raw_cache_path))
+
+
+@skip_pypy
+def test_permission_error(monkeypatch):
+    def save(*args, **kwargs):
+        was_called[0] = True  # Python 2... Use nonlocal instead
+        raise PermissionError
+
+    was_called = [False]
+
+    monkeypatch.setattr(cache, '_save_to_file_system', save)
+    with pytest.warns(Warning):
+        parse(path=__file__, cache=True, diff_cache=True)
+    assert was_called[0]

test/test_diff_parser.py

@@ -8,7 +8,7 @@ import pytest
 from parso.utils import split_lines
 from parso import cache
 from parso import load_grammar
-from parso.python.diff import DiffParser, _assert_valid_graph
+from parso.python.diff import DiffParser, _assert_valid_graph, _assert_nodes_are_equal
 from parso import parse
 
 ANY = object()
@@ -69,6 +69,9 @@ class Differ(object):
 
         _assert_valid_graph(new_module)
 
+        without_diff_parser_module = parse(code)
+        _assert_nodes_are_equal(new_module, without_diff_parser_module)
+
         error_node = _check_error_leaves_nodes(new_module)
         assert expect_error_leaves == (error_node is not None), error_node
         if parsers is not ANY:
@@ -88,15 +91,15 @@ def test_change_and_undo(differ):
     # Parse the function and a.
     differ.initialize(func_before + 'a')
     # Parse just b.
-    differ.parse(func_before + 'b', copies=1, parsers=1)
+    differ.parse(func_before + 'b', copies=1, parsers=2)
     # b has changed to a again, so parse that.
-    differ.parse(func_before + 'a', copies=1, parsers=1)
+    differ.parse(func_before + 'a', copies=1, parsers=2)
     # Same as before parsers should not be used. Just a simple copy.
     differ.parse(func_before + 'a', copies=1)
 
     # Now that we have a newline at the end, everything is easier in Python
     # syntax, we can parse once and then get a copy.
-    differ.parse(func_before + 'a\n', copies=1, parsers=1)
+    differ.parse(func_before + 'a\n', copies=1, parsers=2)
     differ.parse(func_before + 'a\n', copies=1)
 
     # Getting rid of an old parser: Still no parsers used.
@@ -135,7 +138,7 @@ def test_if_simple(differ):
     differ.initialize(src + 'a')
     differ.parse(src + else_ + "a", copies=0, parsers=1)
 
-    differ.parse(else_, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(else_, parsers=2, expect_error_leaves=True)
     differ.parse(src + else_, parsers=1)
 
 
@@ -152,7 +155,7 @@ def test_func_with_for_and_comment(differ):
         # COMMENT
         a""")
     differ.initialize(src)
-    differ.parse('a\n' + src, copies=1, parsers=2)
+    differ.parse('a\n' + src, copies=1, parsers=3)
 
 
 def test_one_statement_func(differ):
@@ -236,7 +239,7 @@ def test_backslash(differ):
         def y():
             pass
     """)
-    differ.parse(src, parsers=2)
+    differ.parse(src, parsers=1)
 
     src = dedent(r"""
     def first():
@@ -247,7 +250,7 @@ def test_backslash(differ):
     def second():
         pass
     """)
-    differ.parse(src, parsers=1)
+    differ.parse(src, parsers=2)
 
 
 def test_full_copy(differ):
@@ -261,10 +264,10 @@ def test_wrong_whitespace(differ):
     hello
     '''
     differ.initialize(code)
-    differ.parse(code + 'bar\n    ', parsers=3)
+    differ.parse(code + 'bar\n    ', parsers=2, expect_error_leaves=True)
 
     code += """abc(\npass\n    """
-    differ.parse(code, parsers=2, copies=1, expect_error_leaves=True)
+    differ.parse(code, parsers=2, expect_error_leaves=True)
 
 
 def test_issues_with_error_leaves(differ):
@@ -279,7 +282,7 @@ def test_issues_with_error_leaves(differ):
         str
     ''')
     differ.initialize(code)
-    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code2, parsers=1, expect_error_leaves=True)
 
 
 def test_unfinished_nodes(differ):
@@ -299,7 +302,7 @@ def test_unfinished_nodes(differ):
     a(1)
     ''')
     differ.initialize(code)
-    differ.parse(code2, parsers=1, copies=2)
+    differ.parse(code2, parsers=2, copies=2)
 
 
 def test_nested_if_and_scopes(differ):
@@ -365,7 +368,7 @@ def test_totally_wrong_whitespace(differ):
     '''
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=4, copies=0, expect_error_leaves=True)
+    differ.parse(code2, parsers=2, copies=0, expect_error_leaves=True)
 
 
 def test_node_insertion(differ):
@@ -439,7 +442,7 @@ def test_in_class_movements(differ):
     """)
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=2, copies=1)
+    differ.parse(code2, parsers=1)
 
 
 def test_in_parentheses_newlines(differ):
@@ -484,7 +487,7 @@ def test_indentation_issue(differ):
     """)
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=1)
+    differ.parse(code2, parsers=2)
 
 
 def test_endmarker_newline(differ):
@@ -585,7 +588,7 @@ def test_if_removal_and_reappearence(differ):
             la
     ''')
     differ.initialize(code1)
-    differ.parse(code2, parsers=1, copies=4, expect_error_leaves=True)
+    differ.parse(code2, parsers=3, copies=2, expect_error_leaves=True)
     differ.parse(code1, parsers=1, copies=1)
     differ.parse(code3, parsers=1, copies=1)
 
@@ -618,8 +621,8 @@ def test_differing_docstrings(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=3, copies=1)
-    differ.parse(code1, parsers=3, copies=1)
+    differ.parse(code2, parsers=2, copies=1)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_one_call_in_function_change(differ):
@@ -649,7 +652,7 @@ def test_one_call_in_function_change(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
     differ.parse(code1, parsers=2, copies=1)
 
 
@@ -711,7 +714,7 @@ def test_docstring_removal(differ):
 
     differ.initialize(code1)
     differ.parse(code2, parsers=1, copies=2)
-    differ.parse(code1, parsers=2, copies=1)
+    differ.parse(code1, parsers=3, copies=1)
 
 
 def test_paren_in_strange_position(differ):
@@ -783,7 +786,7 @@ def test_parentheses_before_method(differ):
 
     differ.initialize(code1)
     differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=1, copies=1)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_indentation_issues(differ):
@@ -824,10 +827,10 @@ def test_indentation_issues(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=2, copies=2, expect_error_leaves=True)
-    differ.parse(code1, copies=2)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
+    differ.parse(code1, copies=1, parsers=2)
     differ.parse(code3, parsers=2, copies=1)
-    differ.parse(code1, parsers=1, copies=2)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_error_dedent_issues(differ):
@@ -860,7 +863,7 @@ def test_error_dedent_issues(differ):
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=6, copies=2, expect_error_leaves=True)
+    differ.parse(code2, parsers=3, copies=0, expect_error_leaves=True)
     differ.parse(code1, parsers=1, copies=0)
 
 
@@ -892,8 +895,8 @@ Some'random text: yeah
         ''')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=1, copies=1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=2, copies=1)
 
 
 def test_many_nested_ifs(differ):
@@ -946,7 +949,7 @@ def test_with_and_funcdef_in_call(differ, prefix):
     code2 = insert_line_into_code(code1, 3, 'def y(self, args):\n')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=3, expect_error_leaves=True)
+    differ.parse(code2, parsers=1, expect_error_leaves=True)
     differ.parse(code1, parsers=1)
 
 
@@ -961,33 +964,30 @@ def test_wrong_backslash(differ):
     code2 = insert_line_into_code(code1, 3, '\\.whl$\n')
 
     differ.initialize(code1)
-    differ.parse(code2, parsers=2, copies=2, expect_error_leaves=True)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
     differ.parse(code1, parsers=1, copies=1)
 
 
-def test_comment_change(differ):
-    differ.initialize('')
-
-
 def test_random_unicode_characters(differ):
     """
     Those issues were all found with the fuzzer.
     """
     differ.initialize('')
-    differ.parse(u'\x1dĔBϞɛˁşʑ˳˻ȣſéÎ\x90̕ȟòwʘ\x1dĔBϞɛˁşʑ˳˻ȣſéÎ', parsers=1, expect_error_leaves=True)
+    differ.parse(u'\x1dĔBϞɛˁşʑ˳˻ȣſéÎ\x90̕ȟòwʘ\x1dĔBϞɛˁşʑ˳˻ȣſéÎ', parsers=1,
+                 expect_error_leaves=True)
     differ.parse(u'\r\r', parsers=1)
     differ.parse(u"˟Ę\x05À\r   rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True)
-    differ.parse(u'a\ntaǁ\rGĒōns__\n\nb', parsers=1)
+    differ.parse(u'a\ntaǁ\rGĒōns__\n\nb', parsers=1,
+                 expect_error_leaves=sys.version_info[0] == 2)
     s = '        if not (self, "_fi\x02\x0e\x08\n\nle"):'
     differ.parse(s, parsers=1, expect_error_leaves=True)
     differ.parse('')
     differ.parse(s + '\n', parsers=1, expect_error_leaves=True)
-    differ.parse(u'   result = (\r\f\x17\t\x11res)', parsers=2, expect_error_leaves=True)
+    differ.parse(u'   result = (\r\f\x17\t\x11res)', parsers=1, expect_error_leaves=True)
     differ.parse('')
-    differ.parse('   a( # xx\ndef', parsers=2, expect_error_leaves=True)
+    differ.parse('   a( # xx\ndef', parsers=1, expect_error_leaves=True)
 
 
-@pytest.mark.skipif(sys.version_info < (2, 7), reason="No set literals in Python 2.6")
 def test_dedent_end_positions(differ):
     code1 = dedent('''\
         if 1:
@@ -996,7 +996,7 @@ def test_dedent_end_positions(differ):
                 c = {
                      5}
         ''')
-    code2 = dedent('''\
+    code2 = dedent(u'''\
         if 1:
             if ⌟ഒᜈྡྷṭb:
                 2
@@ -1039,7 +1039,7 @@ def test_random_character_insertion(differ):
             # 4
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=3, expect_error_leaves=True)
+    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
     differ.parse(code1, copies=1, parsers=1)
 
 
@@ -1100,8 +1100,8 @@ def test_all_sorts_of_indentation(differ):
                 end
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=4, expect_error_leaves=True)
-    differ.parse(code1, copies=1, parsers=3)
+    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True)
 
     code3 = dedent('''\
             if 1:
@@ -1111,7 +1111,7 @@ def test_all_sorts_of_indentation(differ):
                    d
         \x00
         ''')
-    differ.parse(code3, parsers=2, expect_error_leaves=True)
+    differ.parse(code3, parsers=1, expect_error_leaves=True)
     differ.parse('')
 
 
@@ -1128,7 +1128,7 @@ def test_dont_copy_dedents_in_beginning(differ):
         ''')
     differ.initialize(code1)
     differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=2)
+    differ.parse(code1, parsers=1, copies=1)
 
 
 def test_dont_copy_error_leaves(differ):
@@ -1148,7 +1148,7 @@ def test_dont_copy_error_leaves(differ):
         ''')
     differ.initialize(code1)
     differ.parse(code2, parsers=1, expect_error_leaves=True)
-    differ.parse(code1, parsers=2)
+    differ.parse(code1, parsers=1)
 
 
 def test_error_dedent_in_between(differ):
@@ -1172,7 +1172,7 @@ def test_error_dedent_in_between(differ):
             z
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
+    differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True)
     differ.parse(code1, copies=1, parsers=2)
 
 
@@ -1198,8 +1198,8 @@ def test_some_other_indentation_issues(differ):
         a
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=2, parsers=1, expect_error_leaves=True)
-    differ.parse(code1, copies=2, parsers=2)
+    differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, copies=1, parsers=1)
 
 
 def test_open_bracket_case1(differ):
@@ -1239,11 +1239,11 @@ def test_open_bracket_case2(differ):
                 d
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=1, parsers=2, expect_error_leaves=True)
-    differ.parse(code1, copies=2, parsers=0, expect_error_leaves=True)
+    differ.parse(code2, copies=0, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, copies=0, parsers=1, expect_error_leaves=True)
 
 
-def test_x(differ):
+def test_some_weird_removals(differ):
     code1 = dedent('''\
         class C:
             1
@@ -1264,6 +1264,487 @@ def test_x(differ):
             omega
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=ANY, parsers=ANY, expect_error_leaves=True)
-    differ.parse(code3, copies=ANY, parsers=ANY, expect_error_leaves=True)
+    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
+    differ.parse(code3, copies=1, parsers=3, expect_error_leaves=True)
     differ.parse(code1, copies=1)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="Async starts working in 3.5")
+def test_async_copy(differ):
+    code1 = dedent('''\
+        async def main():
+            x = 3
+            print(
+        ''')
+    code2 = dedent('''\
+        async def main():
+            x = 3
+            print()
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, copies=1, parsers=1)
+    differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True)
+
+
+def test_parent_on_decorator(differ):
+    code1 = dedent('''\
+        class AClass:
+            @decorator()
+                def b_test(self):
+                print("Hello")
+                print("world")
+
+            def a_test(self):
+                pass''')
+    code2 = dedent('''\
+        class AClass:
+            @decorator()
+            def b_test(self):
+                print("Hello")
+                print("world")
+
+            def a_test(self):
+                pass''')
+    differ.initialize(code1)
+    module_node = differ.parse(code2, parsers=1)
+    cls = module_node.children[0]
+    cls_suite = cls.children[-1]
+    assert len(cls_suite.children) == 3
+
+
+def test_wrong_indent_in_def(differ):
+    code1 = dedent('''\
+        def x():
+          a
+          b
+        ''')
+
+    code2 = dedent('''\
+        def x():
+         //
+          b
+          c
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=1)
+
+
+def test_backslash_issue(differ):
+    code1 = dedent('''
+        pre = (
+            '')
+        after = 'instead'
+        ''')
+    code2 = dedent('''
+        pre = (
+            '')
+             \\if 
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=1, copies=1)
+
+
+def test_paren_with_indentation(differ):
+    code1 = dedent('''
+        class C:
+            def f(self, fullname, path=None):
+                x
+
+            def load_module(self, fullname):
+                a
+                for prefix in self.search_path:
+                    try:
+                        b
+                    except ImportError:
+                        c
+                else:
+                    raise
+            def x():
+                pass
+        ''')
+    code2 = dedent('''
+        class C:
+            def f(self, fullname, path=None):
+                x
+
+                    (
+                a
+                for prefix in self.search_path:
+                    try:
+                        b
+                    except ImportError:
+                        c
+                else:
+                    raise
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=3, copies=1)
+
+
+def test_error_dedent_in_function(differ):
+    code1 = dedent('''\
+        def x():
+            a
+            b
+            c
+            d
+        ''')
+    code2 = dedent('''\
+        def x():
+            a
+            b
+          c
+            d
+            e
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+
+
+def test_with_formfeed(differ):
+    code1 = dedent('''\
+        @bla
+        async def foo():
+            1
+            yield from []
+            return
+            return ''
+        ''')
+    code2 = dedent('''\
+        @bla
+        async def foo():
+            1
+        \x0cimport 
+            return
+            return ''
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True)
+
+
+def test_repeating_invalid_indent(differ):
+    code1 = dedent('''\
+        def foo():
+            return
+
+        @bla
+            a
+        def foo():
+            a
+            b
+            c
+        ''')
+    code2 = dedent('''\
+        def foo():
+            return
+
+        @bla
+            a
+            b
+            c
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+
+
+def test_another_random_indent(differ):
+    code1 = dedent('''\
+        def foo():
+            a
+        b
+            c
+            return
+        def foo():
+            d
+        ''')
+    code2 = dedent('''\
+        def foo():
+            a
+            c
+            return
+        def foo():
+            d
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=3)
+
+
+def test_invalid_function(differ):
+    code1 = dedent('''\
+        a
+        def foo():
+        def foo():
+            b
+        ''')
+    code2 = dedent('''\
+        a
+        def foo():
+        def foo():
+            b
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_async_func2(differ):
+    code1 = dedent('''\
+        async def foo():
+            return ''
+        @bla
+        async def foo():
+            x
+        ''')
+    code2 = dedent('''\
+        async def foo():
+            return ''
+
+          {
+        @bla
+        async def foo():
+            x
+          y
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True)
+
+
+def test_weird_ending(differ):
+    code1 = dedent('''\
+        def foo():
+            a
+            return
+        ''')
+    code2 = dedent('''\
+        def foo():
+            a
+           nonlocal xF"""
+        y"""''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_nested_class(differ):
+    code1 = dedent('''\
+def c():
+    a = 3
+        class X:
+            b
+        ''')
+    code2 = dedent('''\
+def c():
+    a = 3
+        class X:
+ elif
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_class_with_paren_breaker(differ):
+    code1 = dedent('''\
+class Grammar:
+    x
+    def parse():
+        y
+        parser(
+        )
+        z
+        ''')
+    code2 = dedent('''\
+class Grammar:
+    x
+    def parse():
+        y
+        parser(
+   finally ;
+        )
+        z
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
+
+
+def test_byte_order_mark(differ):
+    code2 = dedent('''\
+
+        x
+        \ufeff
+                 else :
+        ''')
+    differ.initialize('\n')
+    differ.parse(code2, parsers=2, expect_error_leaves=True)
+
+    code3 = dedent('''\
+        \ufeff
+                 if:
+
+        x
+        ''')
+    differ.initialize('\n')
+    differ.parse(code3, parsers=2, expect_error_leaves=True)
+
+
+def test_byte_order_mark2(differ):
+    code = u'\ufeff# foo'
+    differ.initialize(code)
+    differ.parse(code + 'x', parsers=ANY)
+
+
+def test_byte_order_mark3(differ):
+    code1 = u"\ufeff#\ny\n"
+    code2 = u'x\n\ufeff#\n\ufeff#\ny\n'
+    differ.initialize(code1)
+    differ.parse(code2, expect_error_leaves=True, parsers=ANY, copies=ANY)
+    differ.parse(code1, parsers=1)
+
+
+def test_backslash_insertion(differ):
+    code1 = dedent('''
+        def f():
+            x
+            def g():
+                base = "" \\
+                       ""
+                return
+        ''')
+    code2 = dedent('''
+        def f():
+            x
+            def g():
+                base = "" \\
+        def h():
+                       ""
+                return
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)
+    differ.parse(code1, parsers=2, copies=1)
+
+
+def test_fstring_with_error_leaf(differ):
+    code1 = dedent("""\
+        def f():
+            x
+        def g():
+            y
+        """)
+    code2 = dedent("""\
+        def f():
+            x
+            F'''
+        def g():
+            y
+            {a
+        \x01
+        """)
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_yet_another_backslash(differ):
+    code1 = dedent('''\
+        def f():
+            x
+            def g():
+                y
+                base = "" \\
+                       "" % to
+                return
+        ''')
+    code2 = dedent('''\
+        def f():
+            x
+            def g():
+                y
+                base = "" \\
+          \x0f
+                return
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=ANY, copies=ANY, expect_error_leaves=True)
+    differ.parse(code1, parsers=ANY, copies=ANY)
+
+
+def test_backslash_before_def(differ):
+    code1 = dedent('''\
+        def f():
+            x
+
+        def g():
+            y
+            z
+        ''')
+    code2 = dedent('''\
+        def f():
+            x
+         >\\
+        def g():
+            y
+         x
+            z
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=3, copies=1, expect_error_leaves=True)
+
+
+def test_backslash_with_imports(differ):
+    code1 = dedent('''\
+        from x import y, \\
+        ''')
+    code2 = dedent('''\
+        from x import y, \\
+            z
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1)
+    differ.parse(code1, parsers=1)
+
+
+def test_one_line_function_error_recovery(differ):
+    code1 = dedent('''\
+        class X:
+            x
+            def y(): word """
+                # a
+                # b
+                c(self)
+        ''')
+    code2 = dedent('''\
+        class X:
+            x
+            def y(): word """
+                # a
+                # b
+                c(\x01+self)
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=1, copies=1, expect_error_leaves=True)
+
+
+def test_one_line_property_error_recovery(differ):
+    code1 = dedent('''\
+        class X:
+            x
+            @property
+            def encoding(self): True -
+                return 1
+        ''')
+    code2 = dedent('''\
+        class X:
+            x
+            @property
+            def encoding(self): True -
+                return 1
+        ''')
+
+    differ.initialize(code1)
+    differ.parse(code2, parsers=2, copies=1, expect_error_leaves=True)

test/test_error_recovery.py

@@ -1,3 +1,5 @@
+from textwrap import dedent
+
 from parso import parse, load_grammar
 
 
@@ -83,3 +85,65 @@ def test_invalid_token_in_fstr():
     assert error1.type == 'error_leaf'
     assert error2.value == '"'
     assert error2.type == 'error_leaf'
+
+
+def test_dedent_issues1():
+    code = dedent('''\
+        class C:
+            @property
+                f
+                    g
+            end
+        ''')
+    module = load_grammar(version='3.8').parse(code)
+    klass, endmarker = module.children
+    suite = klass.children[-1]
+    assert suite.children[2].type == 'error_leaf'
+    assert suite.children[3].get_code(include_prefix=False) == 'f\n'
+    assert suite.children[5].get_code(include_prefix=False) == 'g\n'
+    assert suite.type == 'suite'
+
+
+def test_dedent_issues2():
+    code = dedent('''\
+        class C:
+            @property
+                if 1:
+                    g
+                else:
+                    h
+            end
+        ''')
+    module = load_grammar(version='3.8').parse(code)
+    klass, endmarker = module.children
+    suite = klass.children[-1]
+    assert suite.children[2].type == 'error_leaf'
+    if_ = suite.children[3]
+    assert if_.children[0] == 'if'
+    assert if_.children[3].type == 'suite'
+    assert if_.children[3].get_code() == '\n            g\n'
+    assert if_.children[4] == 'else'
+    assert if_.children[6].type == 'suite'
+    assert if_.children[6].get_code() == '\n            h\n'
+
+    assert suite.children[4].get_code(include_prefix=False) == 'end\n'
+    assert suite.type == 'suite'
+
+
+def test_dedent_issues3():
+    code = dedent('''\
+        class C:
+          f
+         g
+        ''')
+    module = load_grammar(version='3.8').parse(code)
+    klass, endmarker = module.children
+    suite = klass.children[-1]
+    assert len(suite.children) == 4
+    assert suite.children[1].get_code() == '  f\n'
+    assert suite.children[1].type == 'simple_stmt'
+    assert suite.children[2].get_code() == ''
+    assert suite.children[2].type == 'error_leaf'
+    assert suite.children[2].token_type == 'ERROR_DEDENT'
+    assert suite.children[3].get_code() == ' g\n'
+    assert suite.children[3].type == 'simple_stmt'

test/test_fstring.py

@@ -7,31 +7,62 @@ from parso.python.tokenize import tokenize
 
 @pytest.fixture
 def grammar():
-    return load_grammar(version='3.6')
+    return load_grammar(version='3.8')
 
 
 @pytest.mark.parametrize(
     'code', [
-        '{1}',
-        '{1:}',
-        '',
-        '{1!a}',
-        '{1!a:1}',
-        '{1:1}',
-        '{1:1.{32}}',
-        '{1::>4}',
-        '{foo} {bar}',
+        # simple cases
+        'f"{1}"',
+        'f"""{1}"""',
+        'f"{foo} {bar}"',
+
+        # empty string
+        'f""',
+        'f""""""',
+
+        # empty format specifier is okay
+        'f"{1:}"',
+
+        # use of conversion options
+        'f"{1!a}"',
+        'f"{1!a:1}"',
+
+        # format specifiers
+        'f"{1:1}"',
+        'f"{1:1.{32}}"',
+        'f"{1::>4}"',
+        'f"{x:{y}}"',
+        'f"{x:{y:}}"',
+        'f"{x:{y:1}}"',
 
         # Escapes
-        '{{}}',
-        '{{{1}}}',
-        '{{{1}',
-        '1{{2{{3',
-        '}}',
+        'f"{{}}"',
+        'f"{{{1}}}"',
+        'f"{{{1}"',
+        'f"1{{2{{3"',
+        'f"}}"',
+
+        # New Python 3.8 syntax f'{a=}'
+        'f"{a=}"',
+        'f"{a()=}"',
+
+        # multiline f-string
+        'f"""abc\ndef"""',
+        'f"""abc{\n123}def"""',
+
+        # a line continuation inside of an fstring_string
+        'f"abc\\\ndef"',
+        'f"\\\n{123}\\\n"',
+
+        # a line continuation inside of an fstring_expr
+        'f"{\\\n123}"',
+
+        # a line continuation inside of an format spec
+        'f"{123:.2\\\nf}"',
     ]
 )
 def test_valid(code, grammar):
-    code = 'f"""%s"""' % code
     module = grammar.parse(code, error_recovery=False)
     fstring = module.children[0]
     assert fstring.type == 'fstring'
@@ -40,23 +71,34 @@ def test_valid(code, grammar):
 
 @pytest.mark.parametrize(
     'code', [
-        '}',
-        '{',
-        '{1!{a}}',
-        '{!{a}}',
-        '{}',
-        '{:}',
-        '{:}}}',
-        '{:1}',
-        '{!:}',
-        '{!}',
-        '{!a}',
-        '{1:{}}',
-        '{1:{:}}',
+        # an f-string can't contain unmatched curly braces
+        'f"}"',
+        'f"{"',
+        'f"""}"""',
+        'f"""{"""',
+
+        # invalid conversion characters
+        'f"{1!{a}}"',
+        'f"{!{a}}"',
+
+        # The curly braces must contain an expression
+        'f"{}"',
+        'f"{:}"',
+        'f"{:}}}"',
+        'f"{:1}"',
+        'f"{!:}"',
+        'f"{!}"',
+        'f"{!a}"',
+
+        # invalid (empty) format specifiers
+        'f"{1:{}}"',
+        'f"{1:{:}}"',
+
+        # a newline without a line continuation inside a single-line string
+        'f"abc\ndef"',
     ]
 )
 def test_invalid(code, grammar):
-    code = 'f"""%s"""' % code
     with pytest.raises(ParserSyntaxError):
         grammar.parse(code, error_recovery=False)
 
@@ -79,11 +121,18 @@ def test_tokenize_start_pos(code, positions):
     assert positions == [p.start_pos for p in tokens]
 
 
-def test_roundtrip(grammar):
-    code = dedent("""\
-        f'''s{
-           str.uppe
-        '''
-        """)
+@pytest.mark.parametrize(
+    'code', [
+        dedent("""\
+            f'''s{
+               str.uppe
+            '''
+            """),
+        'f"foo',
+        'f"""foo',
+        'f"abc\ndef"',
+    ]
+)
+def test_roundtrip(grammar, code):
     tree = grammar.parse(code)
     assert tree.get_code() == code

test/test_get_code.py

@@ -118,3 +118,16 @@ def test_carriage_return_at_end(code, types):
     assert tree.get_code() == code
     assert [c.type for c in tree.children] == types
     assert tree.end_pos == (len(code) + 1, 0)
+
+
+@pytest.mark.parametrize('code', [
+    ' ',
+    '    F"""',
+    '    F"""\n',
+    '    F""" \n',
+    '    F""" \n3',
+    '    f"""\n"""',
+    '    f"""\n"""\n',
+])
+def test_full_code_round_trip(code):
+    assert parse(code).get_code() == code

test/test_load_grammar.py

@@ -20,7 +20,7 @@ def test_parse_version(string, result):
     assert utils._parse_version(string) == result
 
 
-@pytest.mark.parametrize('string', ['1.', 'a', '#', '1.3.4.5', '1.12'])
+@pytest.mark.parametrize('string', ['1.', 'a', '#', '1.3.4.5'])
 def test_invalid_grammar_version(string):
     with pytest.raises(ValueError):
         load_grammar(version=string)
@@ -28,4 +28,4 @@ def test_invalid_grammar_version(string):
 
 def test_grammar_int_version():
     with pytest.raises(TypeError):
-        load_grammar(version=3.2)
+        load_grammar(version=3.8)

test/test_normalizer_issues_files.py

@@ -5,9 +5,9 @@ tests of pydocstyle.
 
 import difflib
 import re
+from functools import total_ordering
 
 import parso
-from parso._compatibility import total_ordering
 from parso.utils import python_bytes_to_unicode
 
 

test/test_parser.py

@@ -189,3 +189,35 @@ def test_no_error_nodes(each_version):
                 check(child)
 
     check(parse("if foo:\n bar", version=each_version))
+
+
+def test_named_expression(works_ge_py38):
+    works_ge_py38.parse("(a := 1, a + 1)")
+
+def test_extended_rhs_annassign(works_ge_py38):
+    works_ge_py38.parse("x: y = z,")
+    works_ge_py38.parse("x: Tuple[int, ...] = z, *q, w")
+
+@pytest.mark.parametrize(
+    'param_code', [
+        'a=1, /',
+        'a, /',
+        'a=1, /, b=3',
+        'a, /, b',
+        'a, /, b',
+        'a, /, *, b',
+        'a, /, **kwargs',
+    ]
+)
+def test_positional_only_arguments(works_ge_py38, param_code):
+    works_ge_py38.parse("def x(%s): pass" % param_code)
+
+@pytest.mark.parametrize(
+    'expression', [
+        'a + a',
+        'lambda x: x',
+        'a := lambda x: x'
+    ]
+)
+def test_decorator_expression(works_ge_py39, expression):
+    works_ge_py39.parse("@%s\ndef x(): pass" % expression)

test/test_parser_tree.py

@@ -142,7 +142,7 @@ def test_yields(each_version):
 
 
 def test_yield_from():
-    y, = get_yield_exprs('def x(): (yield from 1)', '3.3')
+    y, = get_yield_exprs('def x(): (yield from 1)', '3.8')
     assert y.type == 'yield_expr'
 
 
@@ -180,3 +180,61 @@ def top_function_three():
 
     r = get_raise_stmts(code, 2) #  Lists inside try-catch
     assert len(list(r)) == 2
+
+
+@pytest.mark.parametrize(
+    'code, name_index, is_definition, include_setitem', [
+        ('x = 3', 0, True, False),
+        ('x.y = 3', 0, False, False),
+        ('x.y = 3', 1, True, False),
+        ('x.y = u.v = z', 0, False, False),
+        ('x.y = u.v = z', 1, True, False),
+        ('x.y = u.v = z', 2, False, False),
+        ('x.y = u.v, w = z', 3, True, False),
+        ('x.y = u.v, w = z', 4, True, False),
+        ('x.y = u.v, w = z', 5, False, False),
+
+        ('x, y = z', 0, True, False),
+        ('x, y = z', 1, True, False),
+        ('x, y = z', 2, False, False),
+        ('x, y = z', 2, False, False),
+        ('x[0], y = z', 2, False, False),
+        ('x[0] = z', 0, False, False),
+        ('x[0], y = z', 0, False, False),
+        ('x[0], y = z', 2, False, True),
+        ('x[0] = z', 0, True, True),
+        ('x[0], y = z', 0, True, True),
+        ('x: int = z', 0, True, False),
+        ('x: int = z', 1, False, False),
+        ('x: int = z', 2, False, False),
+        ('x: int', 0, True, False),
+        ('x: int', 1, False, False),
+    ]
+)
+def test_is_definition(code, name_index, is_definition, include_setitem):
+    module = parse(code, version='3.8')
+    name = module.get_first_leaf()
+    while True:
+        if name.type == 'name':
+            if name_index == 0:
+                break
+            name_index -= 1
+        name = name.get_next_leaf()
+
+    assert name.is_definition(include_setitem=include_setitem) == is_definition
+
+
+def test_iter_funcdefs():
+    code = dedent('''
+        def normal(): ...
+        async def asyn(): ...
+        @dec
+        def dec_normal(): ...
+        @dec1
+        @dec2
+        async def dec_async(): ...
+        def broken
+        ''')
+    module = parse(code, version='3.8')
+    func_names = [f.name.value for f in module.iter_funcdefs()]
+    assert func_names == ['normal', 'asyn', 'dec_normal', 'dec_async']

test/test_pgen2.py

@@ -29,13 +29,17 @@ def _invalid_syntax(code, version=None, **kwargs):
         print(module.children)
 
 
-def test_formfeed(each_py2_version):
-    s = u"""print 1\n\x0Cprint 2\n"""
-    t = _parse(s, each_py2_version)
-    assert t.children[0].children[0].type == 'print_stmt'
-    assert t.children[1].children[0].type == 'print_stmt'
-    s = u"""1\n\x0C\x0C2\n"""
-    t = _parse(s, each_py2_version)
+def test_formfeed(each_version):
+    s = u"foo\n\x0c\nfoo\n"
+    t = _parse(s, each_version)
+    assert t.children[0].children[0].type == 'name'
+    assert t.children[1].children[0].type == 'name'
+    s = u"1\n\x0c\x0c\n2\n"
+    t = _parse(s, each_version)
+
+    with pytest.raises(ParserSyntaxError):
+        s = u"\n\x0c2\n"
+        _parse(s, each_version)
 
 
 def test_matrix_multiplication_operator(works_ge_py35):
@@ -83,6 +87,39 @@ def test_async_for(works_ge_py35):
     works_ge_py35.parse("async def foo():\n async for a in b: pass")
 
 
+@pytest.mark.parametrize("body", [
+    """[1 async for a in b
+    ]""",
+    """[1 async
+    for a in b
+    ]""",
+    """[
+    1
+    async for a in b
+    ]""",
+    """[
+    1
+    async for a
+    in b
+    ]""",
+    """[
+    1
+    async
+    for
+    a
+    in
+    b
+    ]""",
+    """  [
+    1 async for a in b
+  ]""",
+])
+def test_async_for_comprehension_newline(works_ge_py36, body):
+    # Issue #139
+    works_ge_py36.parse("""async def foo():
+    {}""".format(body))
+
+
 def test_async_with(works_ge_py35):
     works_ge_py35.parse("async def foo():\n async with a: pass")
 
@@ -190,6 +227,19 @@ def test_old_octal_notation(works_in_py2):
     works_in_py2.parse("07")
 
 
+def test_long_notation(works_in_py2):
+    works_in_py2.parse("0xFl")
+    works_in_py2.parse("0xFL")
+    works_in_py2.parse("0b1l")
+    works_in_py2.parse("0B1L")
+    works_in_py2.parse("0o7l")
+    works_in_py2.parse("0O7L")
+    works_in_py2.parse("0l")
+    works_in_py2.parse("0L")
+    works_in_py2.parse("10l")
+    works_in_py2.parse("10L")
+
+
 def test_new_binary_notation(each_version):
     _parse("""0b101010""", each_version)
     _invalid_syntax("""0b0101021""", each_version)
@@ -279,12 +329,22 @@ def test_left_recursion():
         generate_grammar('foo: foo NAME\n', tokenize.PythonTokenTypes)
 
 
-def test_ambiguities():
-    with pytest.raises(ValueError, match='ambiguous'):
-        generate_grammar('foo: bar | baz\nbar: NAME\nbaz: NAME\n', tokenize.PythonTokenTypes)
-
-    with pytest.raises(ValueError, match='ambiguous'):
-        generate_grammar('''foo: bar | baz\nbar: 'x'\nbaz: "x"\n''', tokenize.PythonTokenTypes)
-
-    with pytest.raises(ValueError, match='ambiguous'):
-        generate_grammar('''foo: bar | 'x'\nbar: 'x'\n''', tokenize.PythonTokenTypes)
+@pytest.mark.parametrize(
+    'grammar, error_match', [
+        ['foo: bar | baz\nbar: NAME\nbaz: NAME\n',
+         r"foo is ambiguous.*given a TokenType\(NAME\).*bar or baz"],
+        ['''foo: bar | baz\nbar: 'x'\nbaz: "x"\n''',
+         r"foo is ambiguous.*given a ReservedString\(x\).*bar or baz"],
+        ['''foo: bar | 'x'\nbar: 'x'\n''',
+         r"foo is ambiguous.*given a ReservedString\(x\).*bar or foo"],
+        # An ambiguity with the second (not the first) child of a production
+        ['outer: "a" [inner] "b" "c"\ninner: "b" "c" [inner]\n',
+         r"outer is ambiguous.*given a ReservedString\(b\).*inner or outer"],
+        # An ambiguity hidden by a level of indirection (middle)
+        ['outer: "a" [middle] "b" "c"\nmiddle: inner\ninner: "b" "c" [inner]\n',
+         r"outer is ambiguous.*given a ReservedString\(b\).*middle or outer"],
+    ]
+)
+def test_ambiguities(grammar, error_match):
+    with pytest.raises(ValueError, match=error_match):
+        generate_grammar(grammar, tokenize.PythonTokenTypes)

test/test_python_errors.py

@@ -7,6 +7,8 @@ import warnings
 import pytest
 
 import parso
+
+from textwrap import dedent
 from parso._compatibility import is_pypy
 from .failing_examples import FAILING_EXAMPLES, indent, build_nested
 
@@ -37,10 +39,33 @@ def test_python_exception_matches(code):
         error, = errors
         actual = error.message
     assert actual in wanted
-    # Somehow in Python3.3 the SyntaxError().lineno is sometimes None
+    # Somehow in Python2.7 the SyntaxError().lineno is sometimes None
     assert line_nr is None or line_nr == error.start_pos[0]
 
 
+def test_non_async_in_async():
+    """
+    This example doesn't work with FAILING_EXAMPLES, because the line numbers
+    are not always the same / incorrect in Python 3.8.
+    """
+    if sys.version_info[:2] < (3, 5):
+        pytest.skip()
+
+        # Raises multiple errors in previous versions.
+    code = 'async def foo():\n def nofoo():[x async for x in []]'
+    wanted, line_nr = _get_actual_exception(code)
+
+    errors = _get_error_list(code)
+    if errors:
+        error, = errors
+        actual = error.message
+    assert actual in wanted
+    if sys.version_info[:2] < (3, 8):
+        assert line_nr == error.start_pos[0]
+    else:
+        assert line_nr == 0  # For whatever reason this is zero in Python 3.8+
+
+
 @pytest.mark.parametrize(
     ('code', 'positions'), [
         ('1 +', [(1, 3)]),
@@ -95,21 +120,12 @@ def _get_actual_exception(code):
             assert False, "The piece of code should raise an exception."
 
     # SyntaxError
-    # Python 2.6 has a bit different error messages here, so skip it.
-    if sys.version_info[:2] == (2, 6) and wanted == 'SyntaxError: unexpected EOF while parsing':
-        wanted = 'SyntaxError: invalid syntax'
-
     if wanted == 'SyntaxError: non-keyword arg after keyword arg':
         # The python 3.5+ way, a bit nicer.
         wanted = 'SyntaxError: positional argument follows keyword argument'
     elif wanted == 'SyntaxError: assignment to keyword':
-        return [wanted, "SyntaxError: can't assign to keyword"], line_nr
-    elif wanted == 'SyntaxError: assignment to None':
-        # Python 2.6 does has a slightly different error.
-        wanted = 'SyntaxError: cannot assign to None'
-    elif wanted == 'SyntaxError: can not assign to __debug__':
-        # Python 2.6 does has a slightly different error.
-        wanted = 'SyntaxError: cannot assign to __debug__'
+        return [wanted, "SyntaxError: can't assign to keyword",
+                'SyntaxError: cannot assign to __debug__'], line_nr
     elif wanted == 'SyntaxError: can use starred expression only as assignment target':
         # Python 3.4/3.4 have a bit of a different warning than 3.5/3.6 in
         # certain places. But in others this error makes sense.
@@ -171,12 +187,13 @@ def test_statically_nested_blocks():
 
 
 def test_future_import_first():
-    def is_issue(code, *args):
+    def is_issue(code, *args, **kwargs):
         code = code % args
-        return bool(_get_error_list(code))
+        return bool(_get_error_list(code, **kwargs))
 
     i1 = 'from __future__ import division'
     i2 = 'from __future__ import absolute_import'
+    i3 = 'from __future__ import annotations'
     assert not is_issue(i1)
     assert not is_issue(i1 + ';' + i2)
     assert not is_issue(i1 + '\n' + i2)
@@ -187,6 +204,8 @@ def test_future_import_first():
     assert not is_issue('""\n%s;%s', i1, i2)
     assert not is_issue('"";%s;%s ', i1, i2)
     assert not is_issue('"";%s\n%s ', i1, i2)
+    assert not is_issue(i3, version="3.7")
+    assert is_issue(i3, version="3.6")
     assert is_issue('1;' + i1)
     assert is_issue('1\n' + i1)
     assert is_issue('"";1\n' + i1)
@@ -254,6 +273,9 @@ def test_too_many_levels_of_indentation():
     assert not _get_error_list(build_nested('pass', 49, base=base))
     assert _get_error_list(build_nested('pass', 50, base=base))
 
+def test_paren_kwarg():
+    assert _get_error_list("print((sep)=seperator)", version="3.8")
+    assert not _get_error_list("print((sep)=seperator)", version="3.7")
 
 @pytest.mark.parametrize(
     'code', [
@@ -269,6 +291,19 @@ def test_valid_fstrings(code):
     assert not _get_error_list(code, version='3.6')
 
 
+@pytest.mark.parametrize(
+    'code', [
+        'a = (b := 1)',
+        '[x4 := x ** 5 for x in range(7)]',
+        '[total := total + v for v in range(10)]',
+        'while chunk := file.read(2):\n pass',
+        'numbers = [y := math.factorial(x), y**2, y**3]',
+    ]
+)
+def test_valid_namedexpr(code):
+    assert not _get_error_list(code, version='3.8')
+
+
 @pytest.mark.parametrize(
     ('code', 'message'), [
         ("f'{1+}'", ('invalid syntax')),
@@ -283,3 +318,99 @@ def test_invalid_fstrings(code, message):
     """
     error, = _get_error_list(code, version='3.6')
     assert message in error.message
+
+
+@pytest.mark.parametrize(
+    'code', [
+        "from foo import (\nbar,\n rab,\n)",
+        "from foo import (bar, rab, )",
+    ]
+)
+def test_trailing_comma(code):
+    errors = _get_error_list(code)
+    assert not errors
+
+def test_continue_in_finally():
+    code = dedent('''\
+        for a in [1]:
+            try:
+                pass
+            finally:
+                continue
+        ''')
+    assert not _get_error_list(code, version="3.8")
+    assert _get_error_list(code, version="3.7")
+
+    
+@pytest.mark.parametrize(
+    'template', [
+        "a, b, {target}, c = d",
+        "a, b, *{target}, c = d",
+        "(a, *{target}), c = d",
+        "for x, {target} in y: pass",
+        "for x, q, {target} in y: pass",
+        "for x, q, *{target} in y: pass",
+        "for (x, *{target}), q in y: pass",
+    ]
+)
+@pytest.mark.parametrize(
+    'target', [
+        "True",
+        "False",
+        "None",
+        "__debug__"
+    ]
+)
+def test_forbidden_name(template, target):
+    assert _get_error_list(template.format(target=target), version="3")
+
+
+def test_repeated_kwarg():
+    # python 3.9+ shows which argument is repeated
+    assert (
+        _get_error_list("f(q=1, q=2)", version="3.8")[0].message
+        == "SyntaxError: keyword argument repeated"
+    )
+    assert (
+        _get_error_list("f(q=1, q=2)", version="3.9")[0].message
+        == "SyntaxError: keyword argument repeated: q"
+    )
+
+
+@pytest.mark.parametrize(
+    ('source', 'no_errors'), [
+        ('a(a for a in b,)', False),
+        ('a(a for a in b, a)', False),
+        ('a(a, a for a in b)', False),
+        ('a(a, b, a for a in b, c, d)', False),
+        ('a(a for a in b)', True),
+        ('a((a for a in b), c)', True),
+        ('a(c, (a for a in b))', True),
+        ('a(a, b, (a for a in b), c, d)', True),
+    ]
+)
+def test_unparenthesized_genexp(source, no_errors):
+    assert bool(_get_error_list(source)) ^ no_errors
+
+@pytest.mark.parametrize(
+    ('source', 'no_errors'), [
+        ('*x = 2', False),
+        ('(*y) = 1', False),
+        ('((*z)) = 1', False),
+        ('a, *b = 1', True),
+        ('a, *b, c = 1', True),
+        ('a, (*b), c = 1', True),
+        ('a, ((*b)), c = 1', True),
+        ('a, (*b, c), d = 1', True),
+        ('[*(1,2,3)]', True),
+        ('{*(1,2,3)}', True),
+        ('[*(1,2,3),]', True),
+        ('[*(1,2,3), *(4,5,6)]', True),
+        ('[0, *(1,2,3)]', True),
+        ('{*(1,2,3),}', True),
+        ('{*(1,2,3), *(4,5,6)}', True),
+        ('{0, *(4,5,6)}', True)
+    ]
+)
+def test_starred_expr(source, no_errors):
+    assert bool(_get_error_list(source, version="3")) ^ no_errors

test/test_tokenize.py

@@ -1,10 +1,10 @@
 # -*- coding: utf-8    # This file contains Unicode characters.
 
+import sys
 from textwrap import dedent
 
 import pytest
 
-from parso._compatibility import py_version
 from parso.utils import split_lines, parse_version_string
 from parso.python.token import PythonTokenTypes
 from parso.python import tokenize
@@ -16,6 +16,7 @@ from parso.python.tokenize import PythonToken
 NAME = PythonTokenTypes.NAME
 NEWLINE = PythonTokenTypes.NEWLINE
 STRING = PythonTokenTypes.STRING
+NUMBER = PythonTokenTypes.NUMBER
 INDENT = PythonTokenTypes.INDENT
 DEDENT = PythonTokenTypes.DEDENT
 ERRORTOKEN = PythonTokenTypes.ERRORTOKEN
@@ -135,12 +136,12 @@ def test_identifier_contains_unicode():
     ''')
     token_list = _get_token_list(fundef)
     unicode_token = token_list[1]
-    if py_version >= 30:
+    if sys.version_info.major >= 3:
         assert unicode_token[0] == NAME
     else:
         # Unicode tokens in Python 2 seem to be identified as operators.
         # They will be ignored in the parser, that's ok.
-        assert unicode_token[0] == OP
+        assert unicode_token[0] == ERRORTOKEN
 
 
 def test_quoted_strings():
@@ -183,19 +184,19 @@ def test_ur_literals():
             assert typ == NAME
 
     check('u""')
-    check('ur""', is_literal=not py_version >= 30)
-    check('Ur""', is_literal=not py_version >= 30)
-    check('UR""', is_literal=not py_version >= 30)
+    check('ur""', is_literal=not sys.version_info.major >= 3)
+    check('Ur""', is_literal=not sys.version_info.major >= 3)
+    check('UR""', is_literal=not sys.version_info.major >= 3)
     check('bR""')
     # Starting with Python 3.3 this ordering is also possible.
-    if py_version >= 33:
+    if sys.version_info.major >= 3:
         check('Rb""')
 
     # Starting with Python 3.6 format strings where introduced.
-    check('fr""', is_literal=py_version >= 36)
-    check('rF""', is_literal=py_version >= 36)
-    check('f""', is_literal=py_version >= 36)
-    check('F""', is_literal=py_version >= 36)
+    check('fr""', is_literal=sys.version_info >= (3, 6))
+    check('rF""', is_literal=sys.version_info >= (3, 6))
+    check('f""', is_literal=sys.version_info >= (3, 6))
+    check('F""', is_literal=sys.version_info >= (3, 6))
 
 
 def test_error_literal():
@@ -228,22 +229,50 @@ def test_endmarker_end_pos():
     check('a\\')
 
 
+xfail_py2 = dict(marks=[pytest.mark.xfail(sys.version_info[0] == 2, reason='Python 2')])
+
+
 @pytest.mark.parametrize(
     ('code', 'types'), [
+        # Indentation
         (' foo', [INDENT, NAME, DEDENT]),
         ('  foo\n bar', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
         ('  foo\n bar \n baz', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME,
-                                NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
+                                NEWLINE, NAME, DEDENT]),
         (' foo\nbar', [INDENT, NAME, NEWLINE, DEDENT, NAME]),
+
+        # Name stuff
+        ('1foo1', [NUMBER, NAME]),
+        pytest.param(
+            u'மெல்லினம்', [NAME],
+            **xfail_py2),
+        pytest.param(u'²', [ERRORTOKEN], **xfail_py2),
+        pytest.param(u'ä²ö', [NAME, ERRORTOKEN, NAME], **xfail_py2),
+        pytest.param(u'ää²¹öö', [NAME, ERRORTOKEN, NAME], **xfail_py2),
+        (' \x00a', [INDENT, ERRORTOKEN, NAME, DEDENT]),
+        (dedent('''\
+            class BaseCache:
+                    a
+                def
+                    b
+                def
+                    c
+            '''), [NAME, NAME, OP, NEWLINE, INDENT, NAME, NEWLINE,
+                   ERROR_DEDENT, NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT,
+                   NAME, NEWLINE, INDENT, NAME, NEWLINE, DEDENT, DEDENT]),
+        ('  )\n foo', [INDENT, OP, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
+        ('a\n b\n  )\n c', [NAME, NEWLINE, INDENT, NAME, NEWLINE, INDENT, OP,
+                            NEWLINE, DEDENT, NAME, DEDENT]),
+        (' 1 \\\ndef', [INDENT, NUMBER, NAME, DEDENT]),
     ]
 )
-def test_indentation(code, types):
+def test_token_types(code, types):
     actual_types = [t.type for t in _get_token_list(code)]
     assert actual_types == types + [ENDMARKER]
 
 
 def test_error_string():
-    t1, newline, endmarker = _get_token_list(' "\n')
+    indent, t1, newline, token, endmarker = _get_token_list(' "\n')
     assert t1.type == ERRORTOKEN
     assert t1.prefix == ' '
     assert t1.string == '"'
@@ -304,16 +333,18 @@ def test_brackets_no_indentation():
 
 
 def test_form_feed():
-    error_token, endmarker = _get_token_list(dedent('''\
+    indent, error_token, dedent_, endmarker = _get_token_list(dedent('''\
         \f"""'''))
     assert error_token.prefix == '\f'
     assert error_token.string == '"""'
     assert endmarker.prefix == ''
+    assert indent.type == INDENT
+    assert dedent_.type == DEDENT
 
 
 def test_carriage_return():
     lst = _get_token_list(' =\\\rclass')
-    assert [t.type for t in lst] == [INDENT, OP, DEDENT, NAME, ENDMARKER]
+    assert [t.type for t in lst] == [INDENT, OP, NAME, DEDENT, ENDMARKER]
 
 
 def test_backslash():
@@ -324,21 +355,89 @@ def test_backslash():
 
 @pytest.mark.parametrize(
     ('code', 'types'), [
+        # f-strings
         ('f"', [FSTRING_START]),
         ('f""', [FSTRING_START, FSTRING_END]),
         ('f" {}"', [FSTRING_START, FSTRING_STRING, OP, OP, FSTRING_END]),
         ('f" "{}', [FSTRING_START, FSTRING_STRING, FSTRING_END, OP, OP]),
         (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]),
         (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]),
+
+        # format spec
         (r'f"Some {x:.2f}{y}"', [FSTRING_START, FSTRING_STRING, OP, NAME, OP,
                                  FSTRING_STRING, OP, OP, NAME, OP, FSTRING_END]),
+
+        # multiline f-string
+        ('f"""abc\ndef"""', [FSTRING_START, FSTRING_STRING, FSTRING_END]),
+        ('f"""abc{\n123}def"""', [
+            FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING,
+            FSTRING_END
+        ]),
+
+        # a line continuation inside of an fstring_string
+        ('f"abc\\\ndef"', [
+            FSTRING_START, FSTRING_STRING, FSTRING_END
+        ]),
+        ('f"\\\n{123}\\\n"', [
+            FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING,
+            FSTRING_END
+        ]),
+
+        # a line continuation inside of an fstring_expr
+        ('f"{\\\n123}"', [FSTRING_START, OP, NUMBER, OP, FSTRING_END]),
+
+        # a line continuation inside of an format spec
+        ('f"{123:.2\\\nf}"', [
+            FSTRING_START, OP, NUMBER, OP, FSTRING_STRING, OP, FSTRING_END
+        ]),
+
+        # a newline without a line continuation inside a single-line string is
+        # wrong, and will generate an ERRORTOKEN
+        ('f"abc\ndef"', [
+            FSTRING_START, FSTRING_STRING, NEWLINE, NAME, ERRORTOKEN
+        ]),
+
+        # a more complex example
         (r'print(f"Some {x:.2f}a{y}")', [
             NAME, OP, FSTRING_START, FSTRING_STRING, OP, NAME, OP,
             FSTRING_STRING, OP, FSTRING_STRING, OP, NAME, OP, FSTRING_END, OP
         ]),
-
+        # issue #86, a string-like in an f-string expression
+        ('f"{ ""}"', [
+            FSTRING_START, OP, FSTRING_END, STRING
+        ]),
+        ('f"{ f""}"', [
+            FSTRING_START, OP, NAME, FSTRING_END, STRING
+        ]),
     ]
 )
-def test_fstring(code, types, version_ge_py36):
+def test_fstring_token_types(code, types, version_ge_py36):
     actual_types = [t.type for t in _get_token_list(code, version_ge_py36)]
     assert types + [ENDMARKER] == actual_types
+
+
+@pytest.mark.parametrize(
+    ('code', 'types'), [
+        # issue #87, `:=` in the outest paratheses should be tokenized
+        # as a format spec marker and part of the format
+        ('f"{x:=10}"', [
+            FSTRING_START, OP, NAME, OP, FSTRING_STRING, OP, FSTRING_END
+        ]),
+        ('f"{(x:=10)}"', [
+            FSTRING_START, OP, OP, NAME, OP, NUMBER, OP, OP, FSTRING_END
+        ]),
+    ]
+)
+def test_fstring_assignment_expression(code, types, version_ge_py38):
+    actual_types = [t.type for t in _get_token_list(code, version_ge_py38)]
+    assert types + [ENDMARKER] == actual_types
+
+
+def test_fstring_end_error_pos(version_ge_py38):
+    f_start, f_string, bracket, f_end, endmarker = \
+        _get_token_list('f" { "', version_ge_py38)
+    assert f_start.start_pos == (1, 0)
+    assert f_string.start_pos == (1, 2)
+    assert bracket.start_pos == (1, 3)
+    assert f_end.start_pos == (1, 5)
+    assert endmarker.start_pos == (1, 6)

test/test_utils.py

@@ -1,6 +1,11 @@
 from codecs import BOM_UTF8
 
-from parso.utils import split_lines, python_bytes_to_unicode
+from parso.utils import (
+    split_lines,
+    parse_version_string,
+    python_bytes_to_unicode,
+)
+
 import parso
 
 import pytest
@@ -63,3 +68,35 @@ def test_utf8_bom():
     expr_stmt = module.children[0]
     assert expr_stmt.type == 'expr_stmt'
     assert unicode_bom == expr_stmt.get_first_leaf().prefix
+
+
+@pytest.mark.parametrize(
+    ('code', 'errors'), [
+        (b'# coding: wtf-12\nfoo', 'strict'),
+        (b'# coding: wtf-12\nfoo', 'replace'),
+    ]
+)
+def test_bytes_to_unicode_failing_encoding(code, errors):
+    if errors == 'strict':
+        with pytest.raises(LookupError):
+            python_bytes_to_unicode(code, errors=errors)
+    else:
+        python_bytes_to_unicode(code, errors=errors)
+
+@pytest.mark.parametrize(
+    ('version_str', 'version'), [
+        ('3', (3,)),
+        ('3.6', (3, 6)),
+        ('3.6.10', (3, 6)),
+        ('3.10', (3, 10)),
+        ('3.10a9', (3, 10)),
+        ('3.10b9', (3, 10)),
+        ('3.10rc9', (3, 10)),
+    ]
+)
+def test_parse_version_string(version_str, version):
+    parsed_version = parse_version_string(version_str)
+    if len(version) == 1:
+        assert parsed_version[0] == version[0]
+    else:
+        assert parsed_version == version

tox.ini

@@ -1,19 +1,15 @@
 [tox]
-envlist = py27, py33, py34, py35, py36, py37, pypy
+envlist = {py27,py34,py35,py36,py37,py38}
 [testenv]
 extras = testing
 deps =
-    py26,py33: pytest>=3.0.7,<3.3
-    py26,py33: setuptools<37
+    py27,py34: pytest<3.3
+    coverage: coverage
 setenv =
 # https://github.com/tomchristie/django-rest-framework/issues/1957
 # tox corrupts __pycache__, solution from here:
     PYTHONDONTWRITEBYTECODE=1
+    coverage: TOX_TESTENV_COMMAND=coverage run -m pytest
 commands =
-    pytest {posargs:parso test}
-[testenv:cov]
-deps =
-    coverage
-commands =
-    coverage run --source parso -m pytest
-    coverage report
+    {env:TOX_TESTENV_COMMAND:pytest} {posargs}
+    coverage: coverage report