Rename _get_definition to get_definition.

README.rst

@@ -41,7 +41,6 @@ Future
 Known Issues
 ============
 
-- Python3.6's f-strings are not parsed, yet. This means no errors are found in them.
 - `async`/`await` are already used as keywords in Python3.6.
 - `from __future__ import print_function` is not supported,
 

deploy-master.sh

@@ -5,45 +5,48 @@
 set -eu -o pipefail
 
 BASE_DIR=$(dirname $(readlink -f "$0"))
-cd $(BASE_DIR)
+cd $BASE_DIR
+
+git fetch --tags
 
 PROJECT_NAME=parso
 BRANCH=master
 BUILD_FOLDER=build
-FOLDER=$BUILD_FOLDER/$PROJECT_NAME
-
-# Test first.
-#tox
 
 [ -d $BUILD_FOLDER ] || mkdir $BUILD_FOLDER
 # Remove the previous deployment first.
-rm -rf $FOLDER
 # Checkout the right branch
 cd $BUILD_FOLDER
+rm -rf $PROJECT_NAME
 git clone .. $PROJECT_NAME
+cd $PROJECT_NAME
 git checkout $BRANCH
 
+# Test first.
+tox
+
 # Create tag
 tag=v$(python -c "import $PROJECT_NAME; print($PROJECT_NAME.__version__)")
 
-master_ref=$(git show-ref -s $BRANCH)
-tag_ref=$(git show-ref -s $tag | true)
-if [ $tag_ref ]; then
-    if [ $tag_ref != $master_ref ]; then
+master_ref=$(git show-ref -s heads/$BRANCH)
+tag_ref=$(git show-ref -s $tag || true)
+if [[ $tag_ref ]]; then
+    if [[ $tag_ref != $master_ref ]]; then
         echo 'Cannot tag something that has already been tagged with another commit.'
         exit 1
     fi
 else
-    git tag $BRANCH
+    git tag $tag
     git push --tags
 fi
 
 # Package and upload to PyPI
-rm -rf dist/
+#rm -rf dist/ - Not needed anymore, because the folder is never reused.
+echo `pwd`
 python setup.py sdist bdist_wheel
 # Maybe do a pip install twine before.
 twine upload dist/*
 
-cd $(BASE_DIR)
+cd $BASE_DIR
 # Back in the development directory fetch tags.
 git fetch --tags

parso/__init__.py

@@ -23,7 +23,7 @@ from parso.parser import ParserSyntaxError
 from parso.grammar import Grammar, load_grammar
 
 
-__version__ = '0.0.3'
+__version__ = '0.0.4'
 
 
 def parse(code=None, **kwargs):

parso/python/tokenize.py

@@ -62,17 +62,18 @@ def maybe(*choices):
 
 # Return the empty string, plus all of the valid string prefixes.
 def _all_string_prefixes(version_info):
+    def different_case_versions(prefix):
+        for s in _itertools.product(*[(c, c.upper()) for c in prefix]):
+            yield ''.join(s)
     # The valid string prefixes. Only contain the lower case versions,
     #  and don't contain any permuations (include 'fr', but not
     #  'rf'). The various permutations will be generated.
-    _valid_string_prefixes = ['b', 'r', 'u', 'br']
+    _valid_string_prefixes = ['b', 'r', 'u']
+    if version_info >= (3, 0):
+        _valid_string_prefixes.append('br')
+
     if version_info >= (3, 6):
         _valid_string_prefixes += ['f', 'fr']
-    if version_info <= (2, 7):
-        # TODO this is actually not 100% valid. ur is valid in Python 2.7,
-        # while ru is not.
-        # TODO rb is also not valid.
-        _valid_string_prefixes.append('ur')
 
     # if we add binary f-strings, add: ['fb', 'fbr']
     result = set([''])
@@ -80,8 +81,11 @@ def _all_string_prefixes(version_info):
         for t in _itertools.permutations(prefix):
             # create a list with upper and lower versions of each
             #  character
-            for s in _itertools.product(*[(c, c.upper()) for c in t]):
-                result.add(''.join(s))
+            result.update(different_case_versions(t))
+    if version_info <= (2, 7):
+        # In Python 2 the order cannot just be random.
+        result.update(different_case_versions('ur'))
+        result.update(different_case_versions('br'))
     return result
 
 

parso/python/tree.py

@@ -32,6 +32,17 @@ from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf, \
     search_ancestor
 from parso.python.prefix import split_prefix
 
+_FLOW_CONTAINERS = set(['if_stmt', 'while_stmt', 'for_stmt', 'try_stmt',
+                        'with_stmt', 'async_stmt', 'suite'])
+_RETURN_STMT_CONTAINERS = set(['suite', 'simple_stmt']) | _FLOW_CONTAINERS
+_FUNC_CONTAINERS = set(['suite', 'simple_stmt', 'decorated']) | _FLOW_CONTAINERS
+_GET_DEFINITION_TYPES = set([
+    'expr_stmt', 'comp_for', 'with_stmt', 'for_stmt', 'import_name',
+    'import_from', 'param'
+])
+_IMPORTS = set(['import_name', 'import_from'])
+
+
 
 class DocstringMixin(object):
     __slots__ = ()
@@ -42,7 +53,7 @@ class DocstringMixin(object):
         """
         if self.type == 'file_input':
             node = self.children[0]
-        elif isinstance(self, ClassOrFunc):
+        elif self.type in ('funcdef', 'classdef'):
             node = self.children[self.children.index(':') + 1]
             if node.type == 'suite':  # Normally a suite
                 node = node.children[1]  # -> NEWLINE stmt
@@ -67,24 +78,6 @@ class PythonMixin(object):
     """
     __slots__ = ()
 
-    def get_definition(self):
-        if self.type in ('newline', 'endmarker'):
-            raise ValueError('Cannot get the indentation of whitespace or indentation.')
-        scope = self
-        while scope.parent is not None:
-            parent = scope.parent
-            if isinstance(scope, (PythonNode, PythonLeaf)) and parent.type not in ('simple_stmt', 'file_input'):
-                if scope.type == 'testlist_comp':
-                    try:
-                        if scope.children[1].type == 'comp_for':
-                            return scope.children[1]
-                    except IndexError:
-                        pass
-                scope = parent
-            else:
-                break
-        return scope
-
     def get_name_of_position(self, position):
         for c in self.children:
             if isinstance(c, Leaf):
@@ -173,21 +166,55 @@ class Name(_LeafWithoutNewlines):
                                    self.line, self.column)
 
     def is_definition(self):
-        if self.parent.type in ('power', 'atom_expr'):
-            # In `self.x = 3` self is not a definition, but x is.
-            return False
+        """
+        Returns True if the name is being defined.
+        """
+        return self.get_definition() is not None
+
+    def get_definition(self, import_name_always=False):
+        """
+        Returns None if there's on definition for a name.
+
+        :param import_name_alway: Specifies if an import name is always a
+            definition. Normally foo in `from foo import bar` is not a
+            definition.
+        """
+        node = self.parent
+        type_ = node.type
+        if type_ in ('power', 'atom_expr'):
+            # In `self.x = 3` self is not a definition, but x is.
+            return None
+
+        if type_ in ('funcdef', 'classdef'):
+            if self == node.name:
+                return node
+            return None
+
+        if type_ in ():
+            if self in node.get_defined_names():
+                return node
+            return None
+
+        if type_ == 'except_clause':
+            # TODO in Python 2 this doesn't work correctly. See grammar file.
+            #      I think we'll just let it be. Python 2 will be gone in a few
+            #      years.
+            if self.get_previous_sibling() == 'as':
+                return node.parent  # The try_stmt.
+            return None
+
+        while node is not None:
+            if node.type == 'suite':
+                return None
+            if node.type in _GET_DEFINITION_TYPES:
+                if self in node.get_defined_names():
+                    return node
+                if import_name_always and node.type in _IMPORTS:
+                    return node
+                return None
+            node = node.parent
+        return None
 
-        stmt = self.get_definition()
-        if stmt.type in ('funcdef', 'classdef', 'param'):
-            return self == stmt.name
-        elif stmt.type == 'for_stmt':
-            return self.start_pos < stmt.children[2].start_pos
-        elif stmt.type == 'try_stmt':
-            return self.get_previous_sibling() == 'as'
-        else:
-            return stmt.type in ('expr_stmt', 'import_name', 'import_from',
-                                 'comp_for', 'with_stmt') \
-                and self in stmt.get_defined_names()
 
 
 class Literal(PythonLeaf):
@@ -279,8 +306,7 @@ class Scope(PythonBaseNode, DocstringMixin):
             for element in children:
                 if element.type in names:
                     yield element
-                if element.type in ('suite', 'simple_stmt', 'decorated') \
-                        or isinstance(element, Flow):
+                if element.type in _FUNC_CONTAINERS:
                     for e in scan(element.children):
                         yield e
 
@@ -315,14 +341,14 @@ class Module(Scope):
         super(Module, self).__init__(children)
         self._used_names = None
 
-    def iter_future_import_names(self):
+    def _iter_future_import_names(self):
         """
         :return list of str: A list of future import names.
         """
-        # TODO this is a strange scan and not fully correct. I think Python's
-        # parser does it in a different way and scans for the first
-        # statement/import with a tokenizer (to check for syntax changes like
-        # the future print statement).
+        # In Python it's not allowed to use future imports after the first
+        # actual (non-future) statement. However this is not a linter here,
+        # just return all future imports. If people want to scan for issues
+        # they should use the API.
         for imp in self.iter_imports():
             if imp.type == 'import_from' and imp.level == 0:
                 for path in imp.get_paths():
@@ -330,13 +356,14 @@ class Module(Scope):
                     if len(names) == 2 and names[0] == '__future__':
                         yield names[1]
 
-    def has_explicit_absolute_import(self):
+    def _has_explicit_absolute_import(self):
         """
         Checks if imports in this module are explicitly absolute, i.e. there
         is a ``__future__`` import.
+        Currently not public, might be in the future.
         :return bool:
         """
-        for name in self.iter_future_import_names():
+        for name in self._iter_future_import_names():
             if name == 'absolute_import':
                 return True
         return False
@@ -518,14 +545,39 @@ class Function(ClassOrFunc):
         """
         Returns a generator of `yield_expr`.
         """
-        # TODO This is incorrect, yields are also possible in a statement.
-        return self._search_in_scope('yield_expr')
+        def scan(children):
+            for element in children:
+                if element.type in ('classdef', 'funcdef', 'lambdef'):
+                    continue
+
+                try:
+                    nested_children = element.children
+                except AttributeError:
+                    if element.value == 'yield':
+                        if element.parent.type == 'yield_expr':
+                            yield element.parent
+                        else:
+                            yield element
+                else:
+                    for result in scan(nested_children):
+                        yield result
+
+        return scan(self.children)
 
     def iter_return_stmts(self):
         """
         Returns a generator of `return_stmt`.
         """
-        return self._search_in_scope('return_stmt')
+        def scan(children):
+            for element in children:
+                if element.type == 'return_stmt' \
+                        or element.type == 'keyword' and element.value == 'return':
+                    yield element
+                if element.type in _RETURN_STMT_CONTAINERS:
+                    for e in scan(element.children):
+                        yield e
+
+        return scan(self.children)
 
     def is_generator(self):
         """
@@ -651,6 +703,9 @@ class ForStmt(Flow):
         """
         return self.children[3]
 
+    def get_defined_names(self):
+        return _defined_names(self.children[1])
+
 
 class TryStmt(Flow):
     type = 'try_stmt'
@@ -662,7 +717,6 @@ class TryStmt(Flow):
         Returns ``[None]`` for except clauses without an exception given.
         """
         for node in self.children:
-            # TODO this is not correct. We're not returning an except clause.
             if node.type == 'except_clause':
                 yield node.children[1]
             elif node == 'except':
@@ -685,8 +739,7 @@ class WithStmt(Flow):
                 names += _defined_names(with_item.children[2])
         return names
 
-    def get_context_manager_from_name(self, name):
-        # TODO Replace context_manager with test?
+    def get_test_node_from_name(self, name):
         node = name.parent
         if node.type != 'with_item':
             raise ValueError('The name is not actually part of a with statement.')
@@ -1025,6 +1078,9 @@ class Param(PythonBaseNode):
         else:
             return self._tfpdef()
 
+    def get_defined_names(self):
+        return [self.name]
+
     @property
     def position_index(self):
         """

pytest.ini

@@ -2,7 +2,7 @@
 addopts = --doctest-modules
 
 # Ignore broken files inblackbox test directories
-norecursedirs = .* docs scripts normalizer_issue_files
+norecursedirs = .* docs scripts normalizer_issue_files build
 
 # Activate `clean_jedi_cache` fixture for all tests.  This should be
 # fine as long as we are using `clean_jedi_cache` as a session scoped

test/test_absolute_import.py

@@ -10,14 +10,14 @@ def test_explicit_absolute_imports():
     Detect modules with ``from __future__ import absolute_import``.
     """
     module = parse("from __future__ import absolute_import")
-    assert module.has_explicit_absolute_import()
+    assert module._has_explicit_absolute_import()
 
 
 def test_no_explicit_absolute_imports():
     """
      Detect modules without ``from __future__ import absolute_import``.
     """
-    assert not parse("1").has_explicit_absolute_import()
+    assert not parse("1")._has_explicit_absolute_import()
 
 
 def test_dont_break_imports_without_namespaces():
@@ -26,4 +26,4 @@ def test_dont_break_imports_without_namespaces():
     assume that all imports have non-``None`` namespaces.
     """
     src = "from __future__ import absolute_import\nimport xyzzy"
-    assert parse(src).has_explicit_absolute_import()
+    assert parse(src)._has_explicit_absolute_import()

test/test_parser_tree.py

@@ -115,3 +115,37 @@ def test_ellipsis_py2(each_py2_version):
     subscript = trailer.children[1]
     assert subscript.type == 'subscript'
     assert [leaf.value for leaf in subscript.children] == ['.', '.', '.']
+
+
+def get_yield_exprs(code, version):
+    return list(parse(code, version=version).children[0].iter_yield_exprs())
+
+
+def get_return_stmts(code):
+    return list(parse(code).children[0].iter_return_stmts())
+
+
+def test_yields(each_version):
+    y, = get_yield_exprs('def x(): yield', each_version)
+    assert y.value == 'yield'
+    assert y.type == 'keyword'
+
+    y, = get_yield_exprs('def x(): (yield 1)', each_version)
+    assert y.type == 'yield_expr'
+
+    y, = get_yield_exprs('def x(): [1, (yield)]', each_version)
+    assert y.type == 'keyword'
+
+
+def test_yield_from():
+    y, = get_yield_exprs('def x(): (yield from 1)', '3.3')
+    assert y.type == 'yield_expr'
+
+
+def test_returns():
+    r, = get_return_stmts('def x(): return')
+    assert r.value == 'return'
+    assert r.type == 'keyword'
+
+    r, = get_return_stmts('def x(): return 1')
+    assert r.type == 'return_stmt'

test/test_pgen2.py

@@ -254,3 +254,19 @@ def test_multiline_str_literals(each_version):
 
 def test_py2_backticks(works_in_py2):
     works_in_py2.parse("`1`")
+
+
+def test_py2_string_prefixes(works_in_py2):
+    works_in_py2.parse("ur'1'")
+    works_in_py2.parse("Ur'1'")
+    works_in_py2.parse("UR'1'")
+    _invalid_syntax("ru'1'", works_in_py2.version)
+
+
+def py_br(each_version):
+    _parse('br""', each_version)
+
+
+def test_py3_rb(works_ge_py3):
+    works_ge_py3.parse("rb'1'")
+    works_ge_py3.parse("RB'1'")

test/test_tokenize.py

@@ -172,9 +172,9 @@ def test_ur_literals():
     check('Ur""', is_literal=not py_version >= 30)
     check('UR""', is_literal=not py_version >= 30)
     check('bR""')
-    # Starting with Python 3.3 this ordering is also possible, but we just
-    # enable it for all versions. It doesn't hurt.
-    check('Rb""')
+    # Starting with Python 3.3 this ordering is also possible.
+    if py_version >= 33:
+        check('Rb""')
     # Starting with Python 3.6 format strings where introduced.
     check('fr""', is_literal=py_version >= 36)
     check('rF""', is_literal=py_version >= 36)