For Python 2.7 and 3.4 pytest 5 doesn't work anymore

Line continuation characters are valid inside of strings, but weren't
handled correctly in certain cases with f-strings, due to some small
tokenizer bugs.

This pull request to address those issues, and adds tests to validate
the new logic.

.coveragerc

@@ -1,4 +1,5 @@
 [run]
+source = parso
 
 [report]
 # Regexes for lines to exclude from consideration

.travis.yml

@@ -1,25 +1,25 @@
+dist: xenial
 language: python
-sudo: false
 python:
-  - 2.6
   - 2.7
   - 3.4
   - 3.5
   - 3.6
-  - pypy
+  - 3.7
+  - 3.8-dev
+  - pypy2.7-6.0
+  - pypy3.5-6.0
 matrix:
   include:
-    - { python: "3.7", dist: xenial, sudo: true }
     - python: 3.5
-      env: TOXENV=cov
-  allow_failures:
-    - env: TOXENV=cov
+      env: TOXENV=py35-coverage
 install:
     - pip install --quiet tox-travis
 script:
     - tox
 after_script:
-    - if [ $TOXENV == "cov" ]; then
+    - |
+      if [ "${TOXENV%-coverage}" == "$TOXENV" ]; then
         pip install --quiet coveralls;
         coveralls;
       fi

CHANGELOG.rst

@@ -3,7 +3,39 @@
 Changelog
 ---------
 
-0.3.2 (2018-01-24)
+0.5.1 (2019-07-13)
+++++++++++++++++++
+
+- Fix: Some unicode identifiers were not correctly tokenized
+- Fix: Line continuations in f-strings are now working
+
+0.5.0 (2019-06-20)
+++++++++++++++++++
+
+- **Breaking Change** comp_for is now called sync_comp_for for all Python
+  versions to be compatible with the Python 3.8 Grammar
+- Added .pyi stubs for a lot of the parso API
+- Small FileIO changes
+
+0.4.0 (2019-04-05)
+++++++++++++++++++
+
+- Python 3.8 support
+- FileIO support, it's now possible to use abstract file IO, support is alpha
+
+0.3.4 (2019-02-13)
++++++++++++++++++++
+
+- Fix an f-string tokenizer error
+
+0.3.3 (2019-02-06)
++++++++++++++++++++
+
+- Fix async errors in the diff parser
+- A fix in iter_errors
+- This is a very small bugfix release
+
+0.3.2 (2019-01-24)
 +++++++++++++++++++
 
 - 20+ bugfixes in the diff parser and 3 in the tokenizer

conftest.py

@@ -14,7 +14,7 @@ from parso.utils import parse_version_string
 collect_ignore = ["setup.py"]
 
 VERSIONS_2 = '2.6', '2.7'
-VERSIONS_3 = '3.3', '3.4', '3.5', '3.6', '3.7'
+VERSIONS_3 = '3.3', '3.4', '3.5', '3.6', '3.7', '3.8'
 
 
 @pytest.fixture(scope='session')
@@ -155,3 +155,9 @@ def works_ge_py3(each_version):
 def works_ge_py35(each_version):
     version_info = parse_version_string(each_version)
     return Checker(each_version, version_info >= (3, 5))
+
+
+@pytest.fixture
+def works_ge_py38(each_version):
+    version_info = parse_version_string(each_version)
+    return Checker(each_version, version_info >= (3, 8))

parso/__init__.py

@@ -43,7 +43,7 @@ from parso.grammar import Grammar, load_grammar
 from parso.utils import split_lines, python_bytes_to_unicode
 
 
-__version__ = '0.3.2'
+__version__ = '0.5.1'
 
 
 def parse(code=None, **kwargs):

parso/__init__.pyi

@@ -0,0 +1,19 @@
+from typing import Any, Optional, Union
+
+from parso.grammar import Grammar as Grammar, load_grammar as load_grammar
+from parso.parser import ParserSyntaxError as ParserSyntaxError
+from parso.utils import python_bytes_to_unicode as python_bytes_to_unicode, split_lines as split_lines
+
+__version__: str = ...
+
+def parse(
+    code: Optional[Union[str, bytes]],
+    *,
+    version: Optional[str] = None,
+    error_recovery: bool = True,
+    path: Optional[str] = None,
+    start_symbol: Optional[str] = None,
+    cache: bool = False,
+    diff_cache: bool = False,
+    cache_path: Optional[str] = None,
+) -> Any: ...

parso/cache.py

@@ -18,7 +18,7 @@ from parso._compatibility import FileNotFoundError
 LOG = logging.getLogger(__name__)
 
 
-_PICKLE_VERSION = 30
+_PICKLE_VERSION = 32
 """
 Version number (integer) for file system cache.
 
@@ -45,6 +45,7 @@ we generate something similar.  See:
 http://docs.python.org/3/library/sys.html#sys.implementation
 """
 
+
 def _get_default_cache_path():
     if platform.system().lower() == 'windows':
         dir_ = os.path.join(os.getenv('LOCALAPPDATA') or '~', 'Parso', 'Parso')
@@ -54,6 +55,7 @@ def _get_default_cache_path():
         dir_ = os.path.join(os.getenv('XDG_CACHE_HOME') or '~/.cache', 'parso')
     return os.path.expanduser(dir_)
 
+
 _default_cache_path = _get_default_cache_path()
 """
 The path where the cache is stored.
@@ -76,21 +78,25 @@ class _NodeCacheItem(object):
         self.change_time = change_time
 
 
-def load_module(hashed_grammar, path, cache_path=None):
+def load_module(hashed_grammar, file_io, cache_path=None):
     """
     Returns a module or None, if it fails.
     """
-    try:
-        p_time = os.path.getmtime(path)
-    except FileNotFoundError:
+    p_time = file_io.get_last_modified()
+    if p_time is None:
         return None
 
     try:
-        module_cache_item = parser_cache[hashed_grammar][path]
+        module_cache_item = parser_cache[hashed_grammar][file_io.path]
         if p_time <= module_cache_item.change_time:
             return module_cache_item.node
     except KeyError:
-        return _load_from_file_system(hashed_grammar, path, p_time, cache_path=cache_path)
+        return _load_from_file_system(
+            hashed_grammar,
+            file_io.path,
+            p_time,
+            cache_path=cache_path
+        )
 
 
 def _load_from_file_system(hashed_grammar, path, p_time, cache_path=None):
@@ -121,9 +127,10 @@ def _load_from_file_system(hashed_grammar, path, p_time, cache_path=None):
         return module_cache_item.node
 
 
-def save_module(hashed_grammar, path, module, lines, pickling=True, cache_path=None):
+def save_module(hashed_grammar, file_io, module, lines, pickling=True, cache_path=None):
+    path = file_io.path
     try:
-        p_time = None if path is None else os.path.getmtime(path)
+        p_time = None if path is None else file_io.get_last_modified()
     except OSError:
         p_time = None
         pickling = False

parso/file_io.py

@@ -0,0 +1,35 @@
+import os
+
+
+class FileIO(object):
+    def __init__(self, path):
+        self.path = path
+
+    def read(self):  # Returns bytes/str
+        # We would like to read unicode here, but we cannot, because we are not
+        # sure if it is a valid unicode file. Therefore just read whatever is
+        # here.
+        with open(self.path, 'rb') as f:
+            return f.read()
+
+    def get_last_modified(self):
+        """
+        Returns float - timestamp or None, if path doesn't exist.
+        """
+        try:
+            return os.path.getmtime(self.path)
+        except OSError:
+            # Might raise FileNotFoundError, OSError for Python 2
+            return None
+
+    def __repr__(self):
+        return '%s(%s)' % (self.__class__.__name__, self.path)
+
+
+class KnownContentFileIO(FileIO):
+    def __init__(self, path, content):
+        super(KnownContentFileIO, self).__init__(path)
+        self._content = content
+
+    def read(self):
+        return self._content

parso/grammar.py

@@ -12,6 +12,7 @@ from parso.parser import BaseParser
 from parso.python.parser import Parser as PythonParser
 from parso.python.errors import ErrorFinderConfig
 from parso.python import pep8
+from parso.file_io import FileIO, KnownContentFileIO
 
 _loaded_grammars = {}
 
@@ -56,7 +57,8 @@ class Grammar(object):
         :param str path: The path to the file you want to open. Only needed for caching.
         :param bool cache: Keeps a copy of the parser tree in RAM and on disk
             if a path is given. Returns the cached trees if the corresponding
-            files on disk have not changed.
+            files on disk have not changed. Note that this stores pickle files
+            on your file system (e.g. for Linux in ``~/.cache/parso/``).
         :param bool diff_cache: Diffs the cached python module against the new
             code and tries to parse only the parts that have changed. Returns
             the same (changed) module that is found in cache. Using this option
@@ -77,14 +79,14 @@ class Grammar(object):
 
     def _parse(self, code=None, error_recovery=True, path=None,
                start_symbol=None, cache=False, diff_cache=False,
-               cache_path=None, start_pos=(1, 0)):
+               cache_path=None, file_io=None, start_pos=(1, 0)):
         """
         Wanted python3.5 * operator and keyword only arguments. Therefore just
         wrap it all.
         start_pos here is just a parameter internally used. Might be public
         sometime in the future.
         """
-        if code is None and path is None:
+        if code is None and path is None and file_io is None:
             raise TypeError("Please provide either code or a path.")
 
         if start_symbol is None:
@@ -93,15 +95,19 @@ class Grammar(object):
         if error_recovery and start_symbol != 'file_input':
             raise NotImplementedError("This is currently not implemented.")
 
-        if cache and path is not None:
-            module_node = load_module(self._hashed, path, cache_path=cache_path)
+        if file_io is None:
+            if code is None:
+                file_io = FileIO(path)
+            else:
+                file_io = KnownContentFileIO(path, code)
+
+        if cache and file_io.path is not None:
+            module_node = load_module(self._hashed, file_io, cache_path=cache_path)
             if module_node is not None:
                 return module_node
 
         if code is None:
-            with open(path, 'rb') as f:
-                code = f.read()
-
+            code = file_io.read()
         code = python_bytes_to_unicode(code)
 
         lines = split_lines(code, keepends=True)
@@ -110,7 +116,7 @@ class Grammar(object):
                 raise TypeError("You have to define a diff parser to be able "
                                 "to use this option.")
             try:
-                module_cache_item = parser_cache[self._hashed][path]
+                module_cache_item = parser_cache[self._hashed][file_io.path]
             except KeyError:
                 pass
             else:
@@ -125,7 +131,7 @@ class Grammar(object):
                     old_lines=old_lines,
                     new_lines=lines
                 )
-                save_module(self._hashed, path, new_node, lines,
+                save_module(self._hashed, file_io, new_node, lines,
                             # Never pickle in pypy, it's slow as hell.
                             pickling=cache and not is_pypy,
                             cache_path=cache_path)
@@ -141,7 +147,7 @@ class Grammar(object):
         root_node = p.parse(tokens=tokens)
 
         if cache or diff_cache:
-            save_module(self._hashed, path, root_node, lines,
+            save_module(self._hashed, file_io, root_node, lines,
                         # Never pickle in pypy, it's slow as hell.
                         pickling=cache and not is_pypy,
                         cache_path=cache_path)

parso/grammar.pyi

@@ -0,0 +1,38 @@
+from typing import Any, Callable, Generic, Optional, Sequence, TypeVar, Union
+from typing_extensions import Literal
+
+from parso.utils import PythonVersionInfo
+
+_Token = Any
+_NodeT = TypeVar("_NodeT")
+
+class Grammar(Generic[_NodeT]):
+    _default_normalizer_config: Optional[Any] = ...
+    _error_normalizer_config: Optional[Any] = None
+    _start_nonterminal: str = ...
+    _token_namespace: Optional[str] = None
+    def __init__(
+        self,
+        text: str,
+        tokenizer: Callable[[Sequence[str], int], Sequence[_Token]],
+        parser: Any = ...,
+        diff_parser: Any = ...,
+    ) -> None: ...
+    def parse(
+        self,
+        code: Union[str, bytes] = ...,
+        error_recovery: bool = ...,
+        path: Optional[str] = ...,
+        start_symbol: Optional[str] = ...,
+        cache: bool = ...,
+        diff_cache: bool = ...,
+        cache_path: Optional[str] = ...,
+    ) -> _NodeT: ...
+
+class PythonGrammar(Grammar):
+    version_info: PythonVersionInfo
+    def __init__(self, version_info: PythonVersionInfo, bnf_text: str) -> None: ...
+
+def load_grammar(
+    language: Literal["python"] = "python", version: Optional[str] = ..., path: str = ...
+) -> Grammar: ...

parso/normalizer.py

@@ -147,7 +147,6 @@ class Issue(object):
         return '<%s: %s>' % (self.__class__.__name__, self.code)
 
 
-
 class Rule(object):
     code = None
     message = None

parso/pgen2/__init__.pyi

@@ -0,0 +1 @@
+from parso.pgen2.generator import generate_grammar as generate_grammar

parso/pgen2/generator.pyi

@@ -0,0 +1,38 @@
+from typing import Any, Generic, Mapping, Sequence, Set, TypeVar, Union
+
+from parso.pgen2.grammar_parser import NFAState
+
+_TokenTypeT = TypeVar("_TokenTypeT")
+
+class Grammar(Generic[_TokenTypeT]):
+    nonterminal_to_dfas: Mapping[str, Sequence[DFAState[_TokenTypeT]]]
+    reserved_syntax_strings: Mapping[str, ReservedString]
+    start_nonterminal: str
+    def __init__(
+        self,
+        start_nonterminal: str,
+        rule_to_dfas: Mapping[str, Sequence[DFAState]],
+        reserved_syntax_strings: Mapping[str, ReservedString],
+    ) -> None: ...
+
+class DFAPlan:
+    next_dfa: DFAState
+    dfa_pushes: Sequence[DFAState]
+
+class DFAState(Generic[_TokenTypeT]):
+    from_rule: str
+    nfa_set: Set[NFAState]
+    is_final: bool
+    arcs: Mapping[str, DFAState]  # map from all terminals/nonterminals to DFAState
+    nonterminal_arcs: Mapping[str, DFAState]
+    transitions: Mapping[Union[_TokenTypeT, ReservedString], DFAPlan]
+    def __init__(
+        self, from_rule: str, nfa_set: Set[NFAState], final: NFAState
+    ) -> None: ...
+
+class ReservedString:
+    value: str
+    def __init__(self, value: str) -> None: ...
+    def __repr__(self) -> str: ...
+
+def generate_grammar(bnf_grammar: str, token_namespace: Any) -> Grammar[Any]: ...

parso/pgen2/grammar_parser.pyi

@@ -0,0 +1,20 @@
+from typing import Generator, List, Optional, Tuple
+
+from parso.python.token import TokenType
+
+class GrammarParser:
+    generator: Generator[TokenType, None, None]
+    def __init__(self, bnf_grammar: str) -> None: ...
+    def parse(self) -> Generator[Tuple[NFAState, NFAState], None, None]: ...
+
+class NFAArc:
+    next: NFAState
+    nonterminal_or_string: Optional[str]
+    def __init__(
+        self, next_: NFAState, nonterminal_or_string: Optional[str]
+    ) -> None: ...
+
+class NFAState:
+    from_rule: str
+    arcs: List[NFAArc]
+    def __init__(self, from_rule: str) -> None: ...

parso/python/diff.py

@@ -682,6 +682,8 @@ class _NodesTree(object):
                 last = new_nodes[-1]
                 if last.type == 'decorated':
                     last = last.children[-1]
+                if last.type in ('async_funcdef', 'async_stmt'):
+                    last = last.children[-1]
                 last_line_offset_leaf = last.children[-2].get_last_leaf()
                 assert last_line_offset_leaf == ':'
             else:

parso/python/errors.py

@@ -6,7 +6,6 @@ from contextlib import contextmanager
 
 from parso.normalizer import Normalizer, NormalizerConfig, Issue, Rule
 from parso.python.tree import search_ancestor
-from parso.parser import ParserSyntaxError
 
 _BLOCK_STMTS = ('if_stmt', 'while_stmt', 'for_stmt', 'try_stmt', 'with_stmt')
 _STAR_EXPR_PARENTS = ('testlist_star_expr', 'testlist_comp', 'exprlist')
@@ -17,6 +16,7 @@ ALLOWED_FUTURES = (
     'all_feature_names', 'nested_scopes', 'generators', 'division',
     'absolute_import', 'with_statement', 'print_function', 'unicode_literals',
 )
+_COMP_FOR_TYPES = ('comp_for', 'sync_comp_for')
 
 
 def _iter_stmts(scope):
@@ -35,12 +35,12 @@ def _iter_stmts(scope):
 
 def _get_comprehension_type(atom):
     first, second = atom.children[:2]
-    if second.type == 'testlist_comp' and second.children[1].type == 'comp_for':
+    if second.type == 'testlist_comp' and second.children[1].type in _COMP_FOR_TYPES:
         if first == '[':
             return 'list comprehension'
         else:
             return 'generator expression'
-    elif second.type == 'dictorsetmaker' and second.children[-1].type == 'comp_for':
+    elif second.type == 'dictorsetmaker' and second.children[-1].type in _COMP_FOR_TYPES:
         if second.children[1] == ':':
             return 'dict comprehension'
         else:
@@ -107,6 +107,7 @@ def _iter_definition_exprs_from_lists(exprlist):
 
         yield child
 
+
 def _get_expr_stmt_definition_exprs(expr_stmt):
     exprs = []
     for list_ in expr_stmt.children[:-2:2]:
@@ -279,7 +280,6 @@ class ErrorFinder(Normalizer):
                 return ''
         return super(ErrorFinder, self).visit(node)
 
-
     @contextmanager
     def visit_node(self, node):
         self._check_type_rules(node)
@@ -461,17 +461,13 @@ class _YieldFromCheck(SyntaxRule):
 @ErrorFinder.register_rule(type='name')
 class _NameChecks(SyntaxRule):
     message = 'cannot assign to __debug__'
-    message_keyword = 'assignment to keyword'
     message_none = 'cannot assign to None'
 
     def is_issue(self, leaf):
         self._normalizer.context.add_name(leaf)
 
         if leaf.value == '__debug__' and leaf.is_definition():
-            if self._normalizer.version < (3, 0):
             return True
-            else:
-                self.add_issue(leaf, message=self.message_keyword)
         if leaf.value == 'None' and self._normalizer.version < (3, 0) \
                 and leaf.is_definition():
             self.add_issue(leaf, message=self.message_none)
@@ -539,7 +535,7 @@ class _StarStarCheck(SyntaxRule):
     def is_issue(self, leaf):
         if leaf.parent.type == 'dictorsetmaker':
             comp_for = leaf.get_next_sibling().get_next_sibling()
-            return comp_for is not None and comp_for.type == 'comp_for'
+            return comp_for is not None and comp_for.type in _COMP_FOR_TYPES
 
 
 @ErrorFinder.register_rule(value='yield')
@@ -570,11 +566,14 @@ class _BytesAndStringMix(SyntaxRule):
     message = "cannot mix bytes and nonbytes literals"
 
     def _is_bytes_literal(self, string):
+        if string.type == 'fstring':
+            return False
         return 'b' in string.string_prefix.lower()
 
     def is_issue(self, node):
         first = node.children[0]
-        if first.type == 'string' and self._normalizer.version >= (3, 0):
+        # In Python 2 it's allowed to mix bytes and unicode.
+        if self._normalizer.version >= (3, 0):
             first_is_bytes = self._is_bytes_literal(first)
             for string in node.children[1:]:
                 if first_is_bytes != self._is_bytes_literal(string):
@@ -615,7 +614,7 @@ class _FutureImportRule(SyntaxRule):
                     allowed_futures.append('generator_stop')
 
                 if name == 'braces':
-                    self.add_issue(node, message = "not a chance")
+                    self.add_issue(node, message="not a chance")
                 elif name == 'barry_as_FLUFL':
                     m = "Seriously I'm not implementing this :) ~ Dave"
                     self.add_issue(node, message=m)
@@ -635,7 +634,7 @@ class _StarExprRule(SyntaxRule):
             return True
         if node.parent.type == 'testlist_comp':
             # [*[] for a in [1]]
-            if node.parent.children[1].type == 'comp_for':
+            if node.parent.children[1].type in _COMP_FOR_TYPES:
                 self.add_issue(node, message=self.message_iterable_unpacking)
         if self._normalizer.version <= (3, 4):
             n = search_ancestor(node, 'for_stmt', 'expr_stmt')
@@ -728,10 +727,16 @@ class _ArgumentRule(SyntaxRule):
         if node.children[1] == '=' and first.type != 'name':
             if first.type == 'lambdef':
                 # f(lambda: 1=1)
+                if self._normalizer.version < (3, 8):
                     message = "lambda cannot contain assignment"
+                else:
+                    message = 'expression cannot contain assignment, perhaps you meant "=="?'
             else:
                 # f(+x=1)
+                if self._normalizer.version < (3, 8):
                     message = "keyword can't be an expression"
+                else:
+                    message = 'expression cannot contain assignment, perhaps you meant "=="?'
             self.add_issue(first, message=message)
 
 
@@ -755,7 +760,7 @@ class _ArglistRule(SyntaxRule):
     def is_issue(self, node):
         first_arg = node.children[0]
         if first_arg.type == 'argument' \
-                and first_arg.children[1].type == 'comp_for':
+                and first_arg.children[1].type in _COMP_FOR_TYPES:
             # e.g. foo(x for x in [], b)
             return len(node.children) >= 2
         else:
@@ -784,7 +789,8 @@ class _ArglistRule(SyntaxRule):
                         if first == '*':
                             if kw_unpacking_only:
                                 # foo(**kwargs, *args)
-                                message = "iterable argument unpacking follows keyword argument unpacking"
+                                message = "iterable argument unpacking " \
+                                          "follows keyword argument unpacking"
                                 self.add_issue(argument, message=message)
                         else:
                             kw_unpacking_only = True
@@ -806,6 +812,7 @@ class _ArglistRule(SyntaxRule):
                         message = "positional argument follows keyword argument"
                         self.add_issue(argument, message=message)
 
+
 @ErrorFinder.register_rule(type='parameters')
 @ErrorFinder.register_rule(type='lambdef')
 class _ParameterRule(SyntaxRule):
@@ -886,7 +893,13 @@ class _CheckAssignmentRule(SyntaxRule):
             error = _get_comprehension_type(node)
             if error is None:
                 if second.type == 'dictorsetmaker':
+                    if self._normalizer.version < (3, 8):
                         error = 'literal'
+                    else:
+                        if second.children[1] == ':':
+                            error = 'dict display'
+                        else:
+                            error = 'set display'
                 elif first in ('(', '['):
                     if second.type == 'yield_expr':
                         error = 'yield expression'
@@ -898,7 +911,10 @@ class _CheckAssignmentRule(SyntaxRule):
                     else:  # Everything handled, must be useless brackets.
                         self._check_assignment(second, is_deletion)
         elif type_ == 'keyword':
+            if self._normalizer.version < (3, 8):
                 error = 'keyword'
+            else:
+                error = str(node.value)
         elif type_ == 'operator':
             if node.value == '...':
                 error = 'Ellipsis'
@@ -932,21 +948,22 @@ class _CheckAssignmentRule(SyntaxRule):
             error = 'operator'
 
         if error is not None:
-            message = "can't %s %s" % ("delete" if is_deletion else "assign to", error)
+            cannot = "can't" if self._normalizer.version < (3, 8) else "cannot"
+            message = ' '.join([cannot, "delete" if is_deletion else "assign to", error])
             self.add_issue(node, message=message)
 
 
-@ErrorFinder.register_rule(type='comp_for')
+@ErrorFinder.register_rule(type='sync_comp_for')
 class _CompForRule(_CheckAssignmentRule):
     message = "asynchronous comprehension outside of an asynchronous function"
 
     def is_issue(self, node):
-        # Some of the nodes here are already used, so no else if
-        expr_list = node.children[1 + int(node.children[0] == 'async')]
+        expr_list = node.children[1]
+        print(expr_list)
         if expr_list.type != 'expr_list':  # Already handled.
             self._check_assignment(expr_list)
 
-        return node.children[0] == 'async' \
+        return node.parent.children[0] == 'async' \
             and not self._normalizer.context.is_async_funcdef()
 
 

parso/python/grammar27.txt

@@ -107,7 +107,7 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        NAME | NUMBER | strings)
 strings: STRING+
 listmaker: test ( list_for | (',' test)* [','] )
-testlist_comp: test ( comp_for | (',' test)* [','] )
+testlist_comp: test ( sync_comp_for | (',' test)* [','] )
 lambdef: 'lambda' [varargslist] ':' test
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
@@ -115,8 +115,8 @@ subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: expr (',' expr)* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                  (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( (test ':' test (sync_comp_for | (',' test ':' test)* [','])) |
+                  (test (sync_comp_for | (',' test)* [','])) )
 
 classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
 
@@ -125,14 +125,14 @@ arglist: (argument ',')* (argument [',']
                          |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test
+argument: test [sync_comp_for] | test '=' test
 
 list_iter: list_for | list_if
 list_for: 'for' exprlist 'in' testlist_safe [list_iter]
 list_if: 'if' old_test [list_iter]
 
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' old_test [comp_iter]
 
 testlist1: test (',' test)*

parso/python/grammar33.txt

@@ -105,15 +105,15 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
 strings: STRING+
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (test|star_expr) ( sync_comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                  (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( (test ':' test (sync_comp_for | (',' test ':' test)* [','])) |
+                  (test (sync_comp_for | (',' test)* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -122,9 +122,9 @@ arglist: (argument ',')* (argument [',']
                          |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+argument: test [sync_comp_for] | test '=' test  # Really [keyword '='] test
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar34.txt

@@ -105,15 +105,15 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
 strings: STRING+
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (test|star_expr) ( sync_comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
-                  (test (comp_for | (',' test)* [','])) )
+dictorsetmaker: ( (test ':' test (sync_comp_for | (',' test ':' test)* [','])) |
+                  (test (sync_comp_for | (',' test)* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -122,9 +122,9 @@ arglist: (argument ',')* (argument [',']
                          |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+argument: test [sync_comp_for] | test '=' test  # Really [keyword '='] test
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar35.txt

@@ -112,7 +112,7 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
 strings: STRING+
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (test|star_expr) ( sync_comp_for | (',' (test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
@@ -120,9 +120,9 @@ sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
 dictorsetmaker: ( ((test ':' test | '**' expr)
-                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                   (sync_comp_for | (',' (test ':' test | '**' expr))* [','])) |
                   ((test | star_expr)
-                   (comp_for | (',' (test | star_expr))* [','])) )
+                   (sync_comp_for | (',' (test | star_expr))* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
@@ -137,13 +137,13 @@ arglist: argument (',' argument)*  [',']
 # Illegal combinations and orderings are blocked in ast.c:
 # multiple (test comp_for) arguments are blocked; keyword unpackings
 # that precede iterable unpackings are blocked; etc.
-argument: ( test [comp_for] |
+argument: ( test [sync_comp_for] |
             test '=' test |
             '**' test |
             '*' test )
 
-comp_iter: comp_for | comp_if
-comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_iter: sync_comp_for | comp_if
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar36.txt

@@ -140,7 +140,8 @@ argument: ( test [comp_for] |
             '*' test )
 
 comp_iter: comp_for | comp_if
-comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar37.txt

@@ -138,7 +138,8 @@ argument: ( test [comp_for] |
             '*' test )
 
 comp_iter: comp_for | comp_if
-comp_for: ['async'] 'for' exprlist 'in' or_test [comp_iter]
+sync_comp_for: 'for' exprlist 'in' or_test [comp_iter]
+comp_for: ['async'] sync_comp_for
 comp_if: 'if' test_nocond [comp_iter]
 
 # not used in grammar, but may appear in "node" passed from Parser to Compiler

parso/python/grammar38.txt

@@ -20,13 +20,25 @@ async_funcdef: 'async' funcdef
 funcdef: 'def' NAME parameters ['->' test] ':' suite
 
 parameters: '(' [typedargslist] ')'
-typedargslist: (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
+typedargslist: (
+  (tfpdef ['=' test] (',' tfpdef ['=' test])* ',' '/' [',' [ tfpdef ['=' test] (
+        ',' tfpdef ['=' test])* ([',' [
+        '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
+      | '**' tfpdef [',']]])
+  | '*' [tfpdef] (',' tfpdef ['=' test])* ([',' ['**' tfpdef [',']]])
+  | '**' tfpdef [',']]] )
+|  (tfpdef ['=' test] (',' tfpdef ['=' test])* [',' [
         '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
       | '**' tfpdef [',']]]
   | '*' [tfpdef] (',' tfpdef ['=' test])* [',' ['**' tfpdef [',']]]
   | '**' tfpdef [','])
+)
 tfpdef: NAME [':' test]
-varargslist: (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+varargslist: vfpdef ['=' test ](',' vfpdef ['=' test])* ',' '/' [',' [ (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
+        '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+      | '**' vfpdef [',']]]
+  | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
+  | '**' vfpdef [',']) ]] | (vfpdef ['=' test] (',' vfpdef ['=' test])* [',' [
         '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
       | '**' vfpdef [',']]]
   | '*' [vfpdef] (',' vfpdef ['=' test])* [',' ['**' vfpdef [',']]]
@@ -69,8 +81,8 @@ assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt
 async_stmt: 'async' (funcdef | with_stmt | for_stmt)
-if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
-while_stmt: 'while' test ':' suite ['else' ':' suite]
+if_stmt: 'if' namedexpr_test ':' suite ('elif' namedexpr_test ':' suite)* ['else' ':' suite]
+while_stmt: 'while' namedexpr_test ':' suite ['else' ':' suite]
 for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
 try_stmt: ('try' ':' suite
            ((except_clause ':' suite)+
@@ -83,6 +95,7 @@ with_item: test ['as' expr]
 except_clause: 'except' [test ['as' NAME]]
 suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
 
+namedexpr_test: test [':=' test]
 test: or_test ['if' or_test 'else' test] | lambdef
 test_nocond: or_test | lambdef_nocond
 lambdef: 'lambda' [varargslist] ':' test
@@ -108,7 +121,7 @@ atom: ('(' [yield_expr|testlist_comp] ')' |
        '[' [testlist_comp] ']' |
        '{' [dictorsetmaker] '}' |
        NAME | NUMBER | strings | '...' | 'None' | 'True' | 'False')
-testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
+testlist_comp: (namedexpr_test|star_expr) ( comp_for | (',' (namedexpr_test|star_expr))* [','] )
 trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 subscriptlist: subscript (',' subscript)* [',']
 subscript: test | [test] ':' [test] [sliceop]
@@ -134,6 +147,7 @@ arglist: argument (',' argument)*  [',']
 # multiple (test comp_for) arguments are blocked; keyword unpackings
 # that precede iterable unpackings are blocked; etc.
 argument: ( test [comp_for] |
+            test ':=' test |
             test '=' test |
             '**' test |
             '*' test )
@@ -153,5 +167,5 @@ strings: (STRING | fstring)+
 fstring: FSTRING_START fstring_content* FSTRING_END
 fstring_content: FSTRING_STRING | fstring_expr
 fstring_conversion: '!' NAME
-fstring_expr: '{' testlist [ fstring_conversion ] [ fstring_format_spec ] '}'
+fstring_expr: '{' testlist ['='] [ fstring_conversion ] [ fstring_format_spec ] '}'
 fstring_format_spec: ':' fstring_content*

parso/python/parser.py

@@ -39,13 +39,13 @@ class Parser(BaseParser):
         'for_stmt': tree.ForStmt,
         'while_stmt': tree.WhileStmt,
         'try_stmt': tree.TryStmt,
-        'comp_for': tree.CompFor,
+        'sync_comp_for': tree.SyncCompFor,
         # Not sure if this is the best idea, but IMO it's the easiest way to
         # avoid extreme amounts of work around the subtle difference of 2/3
         # grammar in list comoprehensions.
-        'list_for': tree.CompFor,
+        'list_for': tree.SyncCompFor,
         # Same here. This just exists in Python 2.6.
-        'gen_for': tree.CompFor,
+        'gen_for': tree.SyncCompFor,
         'decorator': tree.Decorator,
         'lambdef': tree.Lambda,
         'old_lambdef': tree.Lambda,

parso/python/token.pyi

@@ -0,0 +1,30 @@
+from typing import Container, Iterable
+
+class TokenType:
+    name: str
+    contains_syntax: bool
+    def __init__(self, name: str, contains_syntax: bool) -> None: ...
+
+class TokenTypes:
+    def __init__(
+        self, names: Iterable[str], contains_syntax: Container[str]
+    ) -> None: ...
+
+# not an actual class in the source code, but we need this class to type the fields of
+# PythonTokenTypes
+class _FakePythonTokenTypesClass(TokenTypes):
+    STRING: TokenType
+    NUMBER: TokenType
+    NAME: TokenType
+    ERRORTOKEN: TokenType
+    NEWLINE: TokenType
+    INDENT: TokenType
+    DEDENT: TokenType
+    ERROR_DEDENT: TokenType
+    FSTRING_STRING: TokenType
+    FSTRING_START: TokenType
+    FSTRING_END: TokenType
+    OP: TokenType
+    ENDMARKER: TokenType
+
+PythonTokenTypes: _FakePythonTokenTypesClass = ...

parso/python/tokenize.py

@@ -23,6 +23,9 @@ from parso._compatibility import py_version
 from parso.utils import split_lines
 
 
+# Maximum code point of Unicode 6.0: 0x10ffff (1,114,111)
+MAX_UNICODE = '\U0010ffff'
+
 STRING = PythonTokenTypes.STRING
 NAME = PythonTokenTypes.NAME
 NUMBER = PythonTokenTypes.NUMBER
@@ -51,8 +54,13 @@ if py_version >= 30:
     # Python 3 has str.isidentifier() to check if a char is a valid identifier
     is_identifier = str.isidentifier
 else:
-    namechars = string.ascii_letters + '_'
-    is_identifier = lambda s: s in namechars
+    # Python 2 doesn't, but it's not that important anymore and if you tokenize
+    # Python 2 code with this, it's still ok. It's just that parsing Python 3
+    # code with this function is not 100% correct.
+    # This just means that Python 2 code matches a few identifiers too much,
+    # but that doesn't really matter.
+    def is_identifier(s):
+        return True
 
 
 def group(*choices, **kwargs):
@@ -118,8 +126,10 @@ def _get_token_collection(version_info):
         return result
 
 
-fstring_string_single_line = _compile(r'(?:[^{}\r\n]+|\{\{|\}\})+')
+fstring_string_single_line = _compile(r'(?:\{\{|\}\}|\\(?:\r\n?|\n)|[^{}\r\n])+')
 fstring_string_multi_line = _compile(r'(?:[^{}]+|\{\{|\}\})+')
+fstring_format_spec_single_line = _compile(r'(?:\\(?:\r\n?|\n)|[^{}\r\n])+')
+fstring_format_spec_multi_line = _compile(r'[^{}]+')
 
 
 def _create_token_collection(version_info):
@@ -128,7 +138,16 @@ def _create_token_collection(version_info):
     Whitespace = r'[ \f\t]*'
     whitespace = _compile(Whitespace)
     Comment = r'#[^\r\n]*'
-    Name = r'\w+'
+    # Python 2 is pretty much not working properly anymore, we just ignore
+    # parsing unicode properly, which is fine, I guess.
+    if version_info[0] == 2:
+        Name = r'([A-Za-z_0-9]+)'
+    elif sys.version_info[0] == 2:
+        # Unfortunately the regex engine cannot deal with the regex below, so
+        # just use this one.
+        Name = r'(\w+)'
+    else:
+        Name = u'([A-Za-z_0-9\u0080-' + MAX_UNICODE + ']+)'
 
     if version_info >= (3, 6):
         Hexnumber = r'0[xX](?:_?[0-9a-fA-F])+'
@@ -151,6 +170,8 @@ def _create_token_collection(version_info):
             Octnumber = '0[oO]?[0-7]+'
         Decnumber = r'(?:0+|[1-9][0-9]*)'
         Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+        if version_info[0] < 3:
+            Intnumber += '[lL]?'
         Exponent = r'[eE][-+]?[0-9]+'
         Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
         Expfloat = r'[0-9]+' + Exponent
@@ -186,9 +207,13 @@ def _create_token_collection(version_info):
 
     Bracket = '[][(){}]'
 
-    special_args = [r'\r\n?', r'\n', r'[:;.,@]']
+    special_args = [r'\r\n?', r'\n', r'[;.,@]']
     if version_info >= (3, 0):
         special_args.insert(0, r'\.\.\.')
+    if version_info >= (3, 8):
+        special_args.insert(0, ":=?")
+    else:
+        special_args.insert(0, ":")
     Special = group(*special_args)
 
     Funny = group(Operator, Bracket, Special)
@@ -281,7 +306,10 @@ class FStringNode(object):
         return len(self.quote) == 3
 
     def is_in_expr(self):
-        return (self.parentheses_count - self.format_spec_count) > 0
+        return self.parentheses_count > self.format_spec_count
+
+    def is_in_format_spec(self):
+        return not self.is_in_expr() and self.format_spec_count
 
 
 def _close_fstring_if_necessary(fstring_stack, string, start_pos, additional_prefix):
@@ -303,10 +331,18 @@ def _close_fstring_if_necessary(fstring_stack, string, start_pos, additional_pre
 def _find_fstring_string(endpats, fstring_stack, line, lnum, pos):
     tos = fstring_stack[-1]
     allow_multiline = tos.allow_multiline()
+    if tos.is_in_format_spec():
         if allow_multiline:
-        match = fstring_string_multi_line.match(line, pos)
+            regex = fstring_format_spec_multi_line
         else:
-        match = fstring_string_single_line.match(line, pos)
+            regex = fstring_format_spec_single_line
+    else:
+        if allow_multiline:
+            regex = fstring_string_multi_line
+        else:
+            regex = fstring_string_single_line
+
+    match = regex.match(line, pos)
     if match is None:
         return tos.previous_lines, pos
 
@@ -321,7 +357,9 @@ def _find_fstring_string(endpats, fstring_stack, line, lnum, pos):
 
     new_pos = pos
     new_pos += len(string)
-    if allow_multiline and (string.endswith('\n') or string.endswith('\r')):
+    # even if allow_multiline is False, we still need to check for trailing
+    # newlines, because a single-line f-string can contain line continuations
+    if string.endswith('\n') or string.endswith('\r'):
         tos.previous_lines += string
         string = ''
     else:
@@ -419,8 +457,6 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                 tos = fstring_stack[-1]
                 if not tos.is_in_expr():
                     string, pos = _find_fstring_string(endpats, fstring_stack, line, lnum, pos)
-                    if pos == max:
-                        break
                     if string:
                         yield PythonToken(
                             FSTRING_STRING, string,
@@ -431,6 +467,8 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                         )
                         tos.previous_lines = ''
                         continue
+                    if pos == max:
+                        break
 
                 rest = line[pos:]
                 fstring_end_token, additional_prefix, quote_length = _close_fstring_if_necessary(
@@ -491,6 +529,24 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
             if (initial in numchars or                      # ordinary number
                     (initial == '.' and token != '.' and token != '...')):
                 yield PythonToken(NUMBER, token, spos, prefix)
+            elif pseudomatch.group(3) is not None:            # ordinary name
+                if token in always_break_tokens:
+                    fstring_stack[:] = []
+                    paren_level = 0
+                    # We only want to dedent if the token is on a new line.
+                    if re.match(r'[ \f\t]*$', line[:start]):
+                        while True:
+                            indent = indents.pop()
+                            if indent > start:
+                                yield PythonToken(DEDENT, '', spos, '')
+                            else:
+                                indents.append(indent)
+                                break
+                if is_identifier(token):
+                    yield PythonToken(NAME, token, spos, prefix)
+                else:
+                    for t in _split_illegal_unicode_name(token, spos, prefix):
+                        yield t  # yield from Python 2
             elif initial in '\r\n':
                 if any(not f.allow_multiline() for f in fstring_stack):
                     # Would use fstring_stack.clear, but that's not available
@@ -545,20 +601,6 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
             elif token in fstring_pattern_map:  # The start of an fstring.
                 fstring_stack.append(FStringNode(fstring_pattern_map[token]))
                 yield PythonToken(FSTRING_START, token, spos, prefix)
-            elif is_identifier(initial):                      # ordinary name
-                if token in always_break_tokens:
-                    fstring_stack[:] = []
-                    paren_level = 0
-                    # We only want to dedent if the token is on a new line.
-                    if re.match(r'[ \f\t]*$', line[:start]):
-                        while True:
-                            indent = indents.pop()
-                            if indent > start:
-                                yield PythonToken(DEDENT, '', spos, '')
-                            else:
-                                indents.append(indent)
-                                break
-                yield PythonToken(NAME, token, spos, prefix)
             elif initial == '\\' and line[start:] in ('\\\n', '\\\r\n', '\\\r'):  # continued stmt
                 additional_prefix += prefix + line[start:]
                 break
@@ -575,7 +617,8 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
                         if paren_level:
                             paren_level -= 1
                 elif token == ':' and fstring_stack \
-                        and fstring_stack[-1].parentheses_count == 1:
+                        and fstring_stack[-1].parentheses_count \
+                        - fstring_stack[-1].format_spec_count == 1:
                     fstring_stack[-1].format_spec_count += 1
 
                 yield PythonToken(OP, token, spos, prefix)
@@ -593,6 +636,39 @@ def tokenize_lines(lines, version_info, start_pos=(1, 0)):
     yield PythonToken(ENDMARKER, '', end_pos, additional_prefix)
 
 
+def _split_illegal_unicode_name(token, start_pos, prefix):
+    def create_token():
+        return PythonToken(ERRORTOKEN if is_illegal else NAME, found, pos, prefix)
+
+    found = ''
+    is_illegal = False
+    pos = start_pos
+    for i, char in enumerate(token):
+        if is_illegal:
+            if is_identifier(char):
+                yield create_token()
+                found = char
+                is_illegal = False
+                prefix = ''
+                pos = start_pos[0], start_pos[1] + i
+            else:
+                found += char
+        else:
+            new_found = found + char
+            if is_identifier(new_found):
+                found = new_found
+            else:
+                if found:
+                    yield create_token()
+                    prefix = ''
+                    pos = start_pos[0], start_pos[1] + i
+                found = char
+                is_illegal = True
+
+    if found:
+        yield create_token()
+
+
 if __name__ == "__main__":
     if len(sys.argv) >= 2:
         path = sys.argv[1]

parso/python/tokenize.pyi

@@ -0,0 +1,24 @@
+from typing import Generator, Iterable, NamedTuple, Tuple
+
+from parso.python.token import TokenType
+from parso.utils import PythonVersionInfo
+
+class Token(NamedTuple):
+    type: TokenType
+    string: str
+    start_pos: Tuple[int, int]
+    prefix: str
+    @property
+    def end_pos(self) -> Tuple[int, int]: ...
+
+class PythonToken(Token):
+    def __repr__(self) -> str: ...
+
+def tokenize(
+    code: str, version_info: PythonVersionInfo, start_pos: Tuple[int, int] = (1, 0)
+) -> Generator[PythonToken, None, None]: ...
+def tokenize_lines(
+    lines: Iterable[str],
+    version_info: PythonVersionInfo,
+    start_pos: Tuple[int, int] = (1, 0),
+) -> Generator[PythonToken, None, None]: ...

parso/python/tree.py

@@ -43,6 +43,10 @@ Parser Tree Classes
 """
 
 import re
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
 
 from parso._compatibility import utf8_repr, unicode
 from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf, \
@@ -55,7 +59,7 @@ _FLOW_CONTAINERS = set(['if_stmt', 'while_stmt', 'for_stmt', 'try_stmt',
 _RETURN_STMT_CONTAINERS = set(['suite', 'simple_stmt']) | _FLOW_CONTAINERS
 _FUNC_CONTAINERS = set(['suite', 'simple_stmt', 'decorated']) | _FLOW_CONTAINERS
 _GET_DEFINITION_TYPES = set([
-    'expr_stmt', 'comp_for', 'with_stmt', 'for_stmt', 'import_name',
+    'expr_stmt', 'sync_comp_for', 'with_stmt', 'for_stmt', 'import_name',
     'import_from', 'param'
 ])
 _IMPORTS = set(['import_name', 'import_from'])
@@ -442,7 +446,7 @@ class Module(Scope):
                         recurse(child)
 
             recurse(self)
-            self._used_names = dct
+            self._used_names = UsedNamesMapping(dct)
         return self._used_names
 
 
@@ -466,6 +470,9 @@ class ClassOrFunc(Scope):
         :rtype: list of :class:`Decorator`
         """
         decorated = self.parent
+        if decorated.type == 'async_funcdef':
+            decorated = decorated.parent
+
         if decorated.type == 'decorated':
             if decorated.children[0].type == 'decorators':
                 return decorated.children[0].children
@@ -545,7 +552,8 @@ def _create_params(parent, argslist_list):
                     if param_children[0] == '*' \
                             and (len(param_children) == 1
                                  or param_children[1] == ',') \
-                            or check_python2_nested_param(param_children[0]):
+                            or check_python2_nested_param(param_children[0]) \
+                            or param_children[0] == '/':
                         for p in param_children:
                             p.parent = parent
                         new_children += param_children
@@ -969,7 +977,7 @@ class ImportName(Import):
 class KeywordStatement(PythonBaseNode):
     """
     For the following statements: `assert`, `del`, `global`, `nonlocal`,
-    `raise`, `return`, `yield`, `return`, `yield`.
+    `raise`, `return`, `yield`.
 
     `pass`, `continue` and `break` are not in there, because they are just
     simple keywords and the parser reduces it to a keyword.
@@ -1158,6 +1166,13 @@ class Param(PythonBaseNode):
                 index -= 2
         except ValueError:
             pass
+        try:
+            keyword_only_index = self.parent.children.index('/')
+            if index > keyword_only_index:
+                # Skip the ` /, `
+                index -= 2
+        except ValueError:
+            pass
         return index - 1
 
     def get_parent_function(self):
@@ -1189,8 +1204,8 @@ class Param(PythonBaseNode):
         return '<%s: %s>' % (type(self).__name__, str(self._tfpdef()) + default)
 
 
-class CompFor(PythonBaseNode):
-    type = 'comp_for'
+class SyncCompFor(PythonBaseNode):
+    type = 'sync_comp_for'
     __slots__ = ()
 
     def get_defined_names(self):
@@ -1198,4 +1213,33 @@ class CompFor(PythonBaseNode):
         Returns the a list of `Name` that the comprehension defines.
         """
         # allow async for
-        return _defined_names(self.children[self.children.index('for') + 1])
+        return _defined_names(self.children[1])
+
+
+# This is simply here so an older Jedi version can work with this new parso
+# version. Can be deleted in the next release.
+CompFor = SyncCompFor
+
+
+class UsedNamesMapping(Mapping):
+    """
+    This class exists for the sole purpose of creating an immutable dict.
+    """
+    def __init__(self, dct):
+        self._dict = dct
+
+    def __getitem__(self, key):
+        return self._dict[key]
+
+    def __len__(self):
+        return len(self._dict)
+
+    def __iter__(self):
+        return iter(self._dict)
+
+    def __hash__(self):
+        return id(self)
+
+    def __eq__(self, other):
+        # Comparing these dicts does not make sense.
+        return self is other

parso/utils.pyi

@@ -0,0 +1,29 @@
+from typing import NamedTuple, Optional, Sequence, Union
+
+class Version(NamedTuple):
+    major: int
+    minor: int
+    micro: int
+
+def split_lines(string: str, keepends: bool = ...) -> Sequence[str]: ...
+def python_bytes_to_unicode(
+    source: Union[str, bytes], encoding: str = ..., errors: str = ...
+) -> str: ...
+def version_info() -> Version:
+    """
+    Returns a namedtuple of parso's version, similar to Python's
+    ``sys.version_info``.
+    """
+    ...
+
+class PythonVersionInfo(NamedTuple):
+    major: int
+    minor: int
+
+def parse_version_string(version: Optional[str]) -> PythonVersionInfo:
+    """
+    Checks for a valid version number (e.g. `3.2` or `2.7.1` or `3`) and
+    returns a corresponding version info that is always two characters long in
+    decimal.
+    """
+    ...

pytest.ini

@@ -1,6 +1,8 @@
 [pytest]
 addopts = --doctest-modules
 
+testpaths = parso test
+
 # Ignore broken files inblackbox test directories
 norecursedirs = .* docs scripts normalizer_issue_files build
 

setup.cfg

@@ -1,2 +1,12 @@
 [bdist_wheel]
 universal=1
+
+[flake8]
+max-line-length = 100
+ignore =
+  # do not use bare 'except'
+  E722,
+  # don't know why this was ever even an option, 1+1 should be possible.
+  E226,
+  # line break before binary operator
+  W503,

test/failing_examples.py

@@ -19,14 +19,6 @@ def build_nested(code, depth, base='def f():\n'):
 FAILING_EXAMPLES = [
     '1 +',
     '?',
-    # Python/compile.c
-    dedent('''\
-        for a in [1]:
-            try:
-                pass
-            finally:
-                continue
-        '''), # 'continue' not supported inside 'finally' clause"
     'continue',
     'break',
     'return',
@@ -154,7 +146,7 @@ FAILING_EXAMPLES = [
     # Now nested parsing
     "f'{continue}'",
     "f'{1;1}'",
-    "f'{a=3}'",
+    "f'{a;}'",
     "f'{b\"\" \"\"}'",
 ]
 
@@ -259,10 +251,6 @@ GLOBAL_NONLOCAL_ERROR = [
 
 if sys.version_info >= (3, 6):
     FAILING_EXAMPLES += GLOBAL_NONLOCAL_ERROR
-    FAILING_EXAMPLES += [
-        # Raises multiple errors in previous versions.
-        'async def foo():\n def nofoo():[x async for x in []]',
-    ]
 if sys.version_info >= (3, 5):
     FAILING_EXAMPLES += [
         # Raises different errors so just ignore them for now.
@@ -285,6 +273,14 @@ if sys.version_info >= (3,):
         'b"ä"',
         # combining strings and unicode is allowed in Python 2.
         '"s" b""',
+        '"s" b"" ""',
+        'b"" "" b"" ""',
+    ]
+if sys.version_info >= (3, 6):
+    FAILING_EXAMPLES += [
+        # Same as above, but for f-strings.
+        'f"s" b""',
+        'b"s" f""',
     ]
 if sys.version_info >= (2, 7):
     # This is something that raises a different error in 2.6 than in the other
@@ -311,3 +307,15 @@ if sys.version_info[:2] <= (3, 4):
         'a = *[1], 2',
         '(*[1], 2)',
     ]
+
+if sys.version_info[:2] < (3, 8):
+    FAILING_EXAMPLES += [
+        # Python/compile.c
+        dedent('''\
+            for a in [1]:
+                try:
+                    pass
+                finally:
+                    continue
+            '''),  # 'continue' not supported inside 'finally' clause"
+    ]

test/fuzz_diff_parser.py

@@ -203,9 +203,6 @@ class FileTests:
         self._test_count = test_count
         self._code_lines = self._code_lines
         self._change_count = change_count
-
-        with open(file_path) as f:
-            code = f.read()
         self._file_modifications = []
 
     def _run(self, grammar, file_modifications, debugger, print_code=False):

test/test_cache.py

@@ -10,6 +10,7 @@ from parso.cache import _NodeCacheItem, save_module, load_module, \
     _get_hashed_path, parser_cache, _load_from_file_system, _save_to_file_system
 from parso import load_grammar
 from parso import cache
+from parso import file_io
 
 
 @pytest.fixture()
@@ -76,12 +77,13 @@ def test_modulepickling_simulate_deleted_cache(tmpdir):
     path = tmpdir.dirname + '/some_path'
     with open(path, 'w'):
         pass
+    io = file_io.FileIO(path)
 
-    save_module(grammar._hashed, path, module, [])
-    assert load_module(grammar._hashed, path) == module
+    save_module(grammar._hashed, io, module, lines=[])
+    assert load_module(grammar._hashed, io) == module
 
     unlink(_get_hashed_path(grammar._hashed, path))
     parser_cache.clear()
 
-    cached2 = load_module(grammar._hashed, path)
+    cached2 = load_module(grammar._hashed, io)
     assert cached2 is None

test/test_diff_parser.py

@@ -974,10 +974,12 @@ def test_random_unicode_characters(differ):
     Those issues were all found with the fuzzer.
     """
     differ.initialize('')
-    differ.parse(u'\x1dĔBϞɛˁşʑ˳˻ȣſéÎ\x90̕ȟòwʘ\x1dĔBϞɛˁşʑ˳˻ȣſéÎ', parsers=1, expect_error_leaves=True)
+    differ.parse(u'\x1dĔBϞɛˁşʑ˳˻ȣſéÎ\x90̕ȟòwʘ\x1dĔBϞɛˁşʑ˳˻ȣſéÎ', parsers=1,
+                 expect_error_leaves=True)
     differ.parse(u'\r\r', parsers=1)
     differ.parse(u"˟Ę\x05À\r   rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True)
-    differ.parse(u'a\ntaǁ\rGĒōns__\n\nb', parsers=1)
+    differ.parse(u'a\ntaǁ\rGĒōns__\n\nb', parsers=1,
+                 expect_error_leaves=sys.version_info[0] == 2)
     s = '        if not (self, "_fi\x02\x0e\x08\n\nle"):'
     differ.parse(s, parsers=1, expect_error_leaves=True)
     differ.parse('')
@@ -1243,7 +1245,7 @@ def test_open_bracket_case2(differ):
     differ.parse(code1, copies=2, parsers=0, expect_error_leaves=True)
 
 
-def test_x(differ):
+def test_some_weird_removals(differ):
     code1 = dedent('''\
         class C:
             1
@@ -1264,6 +1266,23 @@ def test_x(differ):
             omega
         ''')
     differ.initialize(code1)
-    differ.parse(code2, copies=ANY, parsers=ANY, expect_error_leaves=True)
-    differ.parse(code3, copies=ANY, parsers=ANY, expect_error_leaves=True)
+    differ.parse(code2, copies=1, parsers=1, expect_error_leaves=True)
+    differ.parse(code3, copies=1, parsers=2, expect_error_leaves=True)
     differ.parse(code1, copies=1)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 5), reason="Async starts working in 3.5")
+def test_async_copy(differ):
+    code1 = dedent('''\
+        async def main():
+            x = 3
+            print(
+        ''')
+    code2 = dedent('''\
+        async def main():
+            x = 3
+            print()
+        ''')
+    differ.initialize(code1)
+    differ.parse(code2, copies=1, parsers=1)
+    differ.parse(code1, copies=1, parsers=1, expect_error_leaves=True)

test/test_fstring.py

@@ -7,31 +7,62 @@ from parso.python.tokenize import tokenize
 
 @pytest.fixture
 def grammar():
-    return load_grammar(version='3.6')
+    return load_grammar(version='3.8')
 
 
 @pytest.mark.parametrize(
     'code', [
-        '{1}',
-        '{1:}',
-        '',
-        '{1!a}',
-        '{1!a:1}',
-        '{1:1}',
-        '{1:1.{32}}',
-        '{1::>4}',
-        '{foo} {bar}',
+        # simple cases
+        'f"{1}"',
+        'f"""{1}"""',
+        'f"{foo} {bar}"',
+
+        # empty string
+        'f""',
+        'f""""""',
+
+        # empty format specifier is okay
+        'f"{1:}"',
+
+        # use of conversion options
+        'f"{1!a}"',
+        'f"{1!a:1}"',
+
+        # format specifiers
+        'f"{1:1}"',
+        'f"{1:1.{32}}"',
+        'f"{1::>4}"',
+        'f"{x:{y}}"',
+        'f"{x:{y:}}"',
+        'f"{x:{y:1}}"',
 
         # Escapes
-        '{{}}',
-        '{{{1}}}',
-        '{{{1}',
-        '1{{2{{3',
-        '}}',
+        'f"{{}}"',
+        'f"{{{1}}}"',
+        'f"{{{1}"',
+        'f"1{{2{{3"',
+        'f"}}"',
+
+        # New Python 3.8 syntax f'{a=}'
+        'f"{a=}"',
+        'f"{a()=}"',
+
+        # multiline f-string
+        'f"""abc\ndef"""',
+        'f"""abc{\n123}def"""',
+
+        # a line continuation inside of an fstring_string
+        'f"abc\\\ndef"',
+        'f"\\\n{123}\\\n"',
+
+        # a line continuation inside of an fstring_expr
+        'f"{\\\n123}"',
+
+        # a line continuation inside of an format spec
+        'f"{123:.2\\\nf}"',
     ]
 )
 def test_valid(code, grammar):
-    code = 'f"""%s"""' % code
     module = grammar.parse(code, error_recovery=False)
     fstring = module.children[0]
     assert fstring.type == 'fstring'
@@ -40,23 +71,34 @@ def test_valid(code, grammar):
 
 @pytest.mark.parametrize(
     'code', [
-        '}',
-        '{',
-        '{1!{a}}',
-        '{!{a}}',
-        '{}',
-        '{:}',
-        '{:}}}',
-        '{:1}',
-        '{!:}',
-        '{!}',
-        '{!a}',
-        '{1:{}}',
-        '{1:{:}}',
+        # an f-string can't contain unmatched curly braces
+        'f"}"',
+        'f"{"',
+        'f"""}"""',
+        'f"""{"""',
+
+        # invalid conversion characters
+        'f"{1!{a}}"',
+        'f"{!{a}}"',
+
+        # The curly braces must contain an expression
+        'f"{}"',
+        'f"{:}"',
+        'f"{:}}}"',
+        'f"{:1}"',
+        'f"{!:}"',
+        'f"{!}"',
+        'f"{!a}"',
+
+        # invalid (empty) format specifiers
+        'f"{1:{}}"',
+        'f"{1:{:}}"',
+
+        # a newline without a line continuation inside a single-line string
+        'f"abc\ndef"',
     ]
 )
 def test_invalid(code, grammar):
-    code = 'f"""%s"""' % code
     with pytest.raises(ParserSyntaxError):
         grammar.parse(code, error_recovery=False)
 
@@ -79,11 +121,18 @@ def test_tokenize_start_pos(code, positions):
     assert positions == [p.start_pos for p in tokens]
 
 
-def test_roundtrip(grammar):
-    code = dedent("""\
+@pytest.mark.parametrize(
+    'code', [
+        dedent("""\
             f'''s{
                str.uppe
             '''
-        """)
+            """),
+        'f"foo',
+        'f"""foo',
+        'f"abc\ndef"',
+    ]
+)
+def test_roundtrip(grammar, code):
     tree = grammar.parse(code)
     assert tree.get_code() == code

test/test_parser.py

@@ -189,3 +189,22 @@ def test_no_error_nodes(each_version):
                 check(child)
 
     check(parse("if foo:\n bar", version=each_version))
+
+
+def test_named_expression(works_ge_py38):
+    works_ge_py38.parse("(a := 1, a + 1)")
+
+
+@pytest.mark.parametrize(
+    'param_code', [
+        'a=1, /',
+        'a, /',
+        'a=1, /, b=3',
+        'a, /, b',
+        'a, /, b',
+        'a, /, *, b',
+        'a, /, **kwargs',
+    ]
+)
+def test_positional_only_arguments(works_ge_py38, param_code):
+    works_ge_py38.parse("def x(%s): pass" % param_code)

test/test_pgen2.py

@@ -190,6 +190,19 @@ def test_old_octal_notation(works_in_py2):
     works_in_py2.parse("07")
 
 
+def test_long_notation(works_in_py2):
+    works_in_py2.parse("0xFl")
+    works_in_py2.parse("0xFL")
+    works_in_py2.parse("0b1l")
+    works_in_py2.parse("0B1L")
+    works_in_py2.parse("0o7l")
+    works_in_py2.parse("0O7L")
+    works_in_py2.parse("0l")
+    works_in_py2.parse("0L")
+    works_in_py2.parse("10l")
+    works_in_py2.parse("10L")
+
+
 def test_new_binary_notation(each_version):
     _parse("""0b101010""", each_version)
     _invalid_syntax("""0b0101021""", each_version)

test/test_python_errors.py

@@ -41,6 +41,29 @@ def test_python_exception_matches(code):
     assert line_nr is None or line_nr == error.start_pos[0]
 
 
+def test_non_async_in_async():
+    """
+    This example doesn't work with FAILING_EXAMPLES, because the line numbers
+    are not always the same / incorrect in Python 3.8.
+    """
+    if sys.version_info[:2] < (3, 5):
+        pytest.skip()
+
+        # Raises multiple errors in previous versions.
+    code = 'async def foo():\n def nofoo():[x async for x in []]'
+    wanted, line_nr = _get_actual_exception(code)
+
+    errors = _get_error_list(code)
+    if errors:
+        error, = errors
+        actual = error.message
+    assert actual in wanted
+    if sys.version_info[:2] < (3, 8):
+        assert line_nr == error.start_pos[0]
+    else:
+        assert line_nr == 0  # For whatever reason this is zero in Python 3.8+
+
+
 @pytest.mark.parametrize(
     ('code', 'positions'), [
         ('1 +', [(1, 3)]),
@@ -103,7 +126,8 @@ def _get_actual_exception(code):
         # The python 3.5+ way, a bit nicer.
         wanted = 'SyntaxError: positional argument follows keyword argument'
     elif wanted == 'SyntaxError: assignment to keyword':
-        return [wanted, "SyntaxError: can't assign to keyword"], line_nr
+        return [wanted, "SyntaxError: can't assign to keyword",
+                'SyntaxError: cannot assign to __debug__'], line_nr
     elif wanted == 'SyntaxError: assignment to None':
         # Python 2.6 does has a slightly different error.
         wanted = 'SyntaxError: cannot assign to None'

test/test_tokenize.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8    # This file contains Unicode characters.
 
+import sys
 from textwrap import dedent
 
 import pytest
@@ -16,6 +17,7 @@ from parso.python.tokenize import PythonToken
 NAME = PythonTokenTypes.NAME
 NEWLINE = PythonTokenTypes.NEWLINE
 STRING = PythonTokenTypes.STRING
+NUMBER = PythonTokenTypes.NUMBER
 INDENT = PythonTokenTypes.INDENT
 DEDENT = PythonTokenTypes.DEDENT
 ERRORTOKEN = PythonTokenTypes.ERRORTOKEN
@@ -140,7 +142,7 @@ def test_identifier_contains_unicode():
     else:
         # Unicode tokens in Python 2 seem to be identified as operators.
         # They will be ignored in the parser, that's ok.
-        assert unicode_token[0] == OP
+        assert unicode_token[0] == ERRORTOKEN
 
 
 def test_quoted_strings():
@@ -228,16 +230,29 @@ def test_endmarker_end_pos():
     check('a\\')
 
 
+xfail_py2 = dict(marks=[pytest.mark.xfail(sys.version_info[0] == 2, reason='Python 2')])
+
+
 @pytest.mark.parametrize(
     ('code', 'types'), [
+        # Indentation
         (' foo', [INDENT, NAME, DEDENT]),
         ('  foo\n bar', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
         ('  foo\n bar \n baz', [INDENT, NAME, NEWLINE, ERROR_DEDENT, NAME,
                                 NEWLINE, ERROR_DEDENT, NAME, DEDENT]),
         (' foo\nbar', [INDENT, NAME, NEWLINE, DEDENT, NAME]),
+
+        # Name stuff
+        ('1foo1', [NUMBER, NAME]),
+        pytest.param(
+            u'மெல்லினம்', [NAME],
+            **xfail_py2),
+        pytest.param(u'²', [ERRORTOKEN], **xfail_py2),
+        pytest.param(u'ä²ö', [NAME, ERRORTOKEN, NAME], **xfail_py2),
+        pytest.param(u'ää²¹öö', [NAME, ERRORTOKEN, NAME], **xfail_py2),
     ]
 )
-def test_indentation(code, types):
+def test_token_types(code, types):
     actual_types = [t.type for t in _get_token_list(code)]
     assert actual_types == types + [ENDMARKER]
 
@@ -330,13 +345,46 @@ def test_backslash():
         ('f" "{}', [FSTRING_START, FSTRING_STRING, FSTRING_END, OP, OP]),
         (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]),
         (r'f"\""', [FSTRING_START, FSTRING_STRING, FSTRING_END]),
+
+        # format spec
         (r'f"Some {x:.2f}{y}"', [FSTRING_START, FSTRING_STRING, OP, NAME, OP,
                                  FSTRING_STRING, OP, OP, NAME, OP, FSTRING_END]),
+
+        # multiline f-string
+        ('f"""abc\ndef"""', [FSTRING_START, FSTRING_STRING, FSTRING_END]),
+        ('f"""abc{\n123}def"""', [
+            FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING,
+            FSTRING_END
+        ]),
+
+        # a line continuation inside of an fstring_string
+        ('f"abc\\\ndef"', [
+            FSTRING_START, FSTRING_STRING, FSTRING_END
+        ]),
+        ('f"\\\n{123}\\\n"', [
+            FSTRING_START, FSTRING_STRING, OP, NUMBER, OP, FSTRING_STRING,
+            FSTRING_END
+        ]),
+
+        # a line continuation inside of an fstring_expr
+        ('f"{\\\n123}"', [FSTRING_START, OP, NUMBER, OP, FSTRING_END]),
+
+        # a line continuation inside of an format spec
+        ('f"{123:.2\\\nf}"', [
+            FSTRING_START, OP, NUMBER, OP, FSTRING_STRING, OP, FSTRING_END
+        ]),
+
+        # a newline without a line continuation inside a single-line string is
+        # wrong, and will generate an ERRORTOKEN
+        ('f"abc\ndef"', [
+            FSTRING_START, FSTRING_STRING, NEWLINE, NAME, ERRORTOKEN
+        ]),
+
+        # a more complex example
         (r'print(f"Some {x:.2f}a{y}")', [
             NAME, OP, FSTRING_START, FSTRING_STRING, OP, NAME, OP,
             FSTRING_STRING, OP, FSTRING_STRING, OP, NAME, OP, FSTRING_END, OP
         ]),
-
     ]
 )
 def test_fstring(code, types, version_ge_py36):

tox.ini

@@ -1,19 +1,17 @@
 [tox]
-envlist = py27, py33, py34, py35, py36, py37, pypy
+envlist = {py26,py27,py33,py34,py35,py36,py37}
 [testenv]
 extras = testing
 deps =
     py26,py33: pytest>=3.0.7,<3.3
+    py27,py34: pytest<5
     py26,py33: setuptools<37
+    coverage: coverage
 setenv =
 # https://github.com/tomchristie/django-rest-framework/issues/1957
 # tox corrupts __pycache__, solution from here:
     PYTHONDONTWRITEBYTECODE=1
+    coverage: TOX_TESTENV_COMMAND=coverage run -m pytest
 commands =
-    pytest {posargs:parso test}
-[testenv:cov]
-deps =
-    coverage
-commands =
-    coverage run --source parso -m pytest
-    coverage report
+    {env:TOX_TESTENV_COMMAND:pytest} {posargs}
+    coverage: coverage report