Apply \r changes in syntax trees

parso/python/parser.py

@@ -127,8 +127,9 @@ class Parser(BaseParser):
             last_leaf = None
 
         if self._start_nonterminal == 'file_input' and \
-                (token.type == PythonTokenTypes.ENDMARKER or
-                 token.type == DEDENT and '\n' not in last_leaf.value):
+                (token.type == PythonTokenTypes.ENDMARKER
+                 or token.type == DEDENT and '\n' not in last_leaf.value
+                 and '\r' not in last_leaf.value):
             # In Python statements need to end with a newline. But since it's
             # possible (and valid in Python ) that there's no newline at the
             # end of a file, we have to recover even if the user doesn't want

parso/python/tree.py

@@ -48,6 +48,7 @@ from parso._compatibility import utf8_repr, unicode
 from parso.tree import Node, BaseNode, Leaf, ErrorNode, ErrorLeaf, \
     search_ancestor
 from parso.python.prefix import split_prefix
+from parso.utils import split_lines
 
 _FLOW_CONTAINERS = set(['if_stmt', 'while_stmt', 'for_stmt', 'try_stmt',
                         'with_stmt', 'async_stmt', 'suite'])
@@ -127,8 +128,10 @@ class PythonLeaf(PythonMixin, Leaf):
                 and previous_leaf.token_type in ('INDENT', 'ERROR_DEDENT'):
             previous_leaf = previous_leaf.get_previous_leaf()
 
-        if previous_leaf is None:
-            return self.line - self.prefix.count('\n'), 0  # It's the first leaf.
+        if previous_leaf is None:  # It's the first leaf.
+            lines = split_lines(self.prefix)
+            # + 1 is needed because split_lines always returns at least [''].
+            return self.line - len(lines) + 1, 0  # It's the first leaf.
         return previous_leaf.end_pos
 
 

parso/tree.py

@@ -1,5 +1,7 @@
 from abc import abstractmethod, abstractproperty
+
 from parso._compatibility import utf8_repr, encoding, py_version
+from parso.utils import split_lines
 
 
 def search_ancestor(node, *node_types):
@@ -193,7 +195,9 @@ class Leaf(NodeOrLeaf):
     def get_start_pos_of_prefix(self):
         previous_leaf = self.get_previous_leaf()
         if previous_leaf is None:
-            return self.line - self.prefix.count('\n'), 0  # It's the first leaf.
+            lines = split_lines(self.prefix)
+            # + 1 is needed because split_lines always returns at least [''].
+            return self.line - len(lines) + 1, 0  # It's the first leaf.
         return previous_leaf.end_pos
 
     def get_first_leaf(self):
@@ -210,7 +214,7 @@ class Leaf(NodeOrLeaf):
 
     @property
     def end_pos(self):
-        lines = self.value.split('\n')
+        lines = split_lines(self.value)
         end_pos_line = self.line + len(lines) - 1
         # Check for multiline token
         if self.line == end_pos_line:
@@ -316,7 +320,7 @@ class BaseNode(NodeOrLeaf):
 
     @utf8_repr
     def __repr__(self):
-        code = self.get_code().replace('\n', ' ').strip()
+        code = self.get_code().replace('\n', ' ').replace('\r', ' ').strip()
         if not py_version >= 30:
             code = code.encode(encoding, 'replace')
         return "<%s: %s@%s,%s>" % \

test/fuzz_diff_parser.py

@@ -155,7 +155,6 @@ class FileTests:
             print()
         except Exception:
             print("Issue in file: %s" % self._path)
-            raise
             if debugger:
                 einfo = sys.exc_info()
                 pdb = __import__(debugger)

test/test_diff_parser.py

@@ -960,4 +960,5 @@ def test_random_unicode_characters(differ):
     differ.parse('\x1dĔBϞɛˁşʑ˳˻ȣſéÎ\x90̕ȟòwʘ\x1dĔBϞɛˁşʑ˳˻ȣſéÎ', parsers=1, expect_error_leaves=True)
     differ.parse('\r\r', parsers=1)
     differ.parse("˟Ę\x05À\r   rúƣ@\x8a\x15r()\n", parsers=1, expect_error_leaves=True)
+    differ.parse('a\ntaǁ\rGĒōns__\n\nb', parsers=1)
     differ.parse('')