From b38d31b99d46815967b56fa81468f4ec12b252ee Mon Sep 17 00:00:00 2001 From: "T.Rzepka" Date: Sun, 11 Feb 2018 22:37:57 +0100 Subject: [PATCH 1/5] Fix for Python 2 on Windows, see #1037. --- AUTHORS.txt | 1 + jedi/evaluate/compiled/subprocess/__init__.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/AUTHORS.txt b/AUTHORS.txt index 691795dd..bce27b07 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -47,5 +47,6 @@ Simon Ruggier (@sruggier) Robin Roth (@robinro) Malte Plath (@langsamer) Anton Zub (@zabulazza) +Tobias Rzepka (@TobiasRzepka) Note: (@user) means a github user name. diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index a8de8792..e4d5a304 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -138,11 +138,13 @@ class _CompiledSubprocess(object): _MAIN_PATH, os.path.dirname(os.path.dirname(parso_path)) ) + universal_newlines = not is_py3 return subprocess.Popen( args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, # stderr=subprocess.PIPE + universal_newlines=universal_newlines ) def run(self, evaluator, function, args=(), kwargs={}): From 446de514021d024ebb1eab3af63930e924ae8a6f Mon Sep 17 00:00:00 2001 From: "T.Rzepka" Date: Sat, 17 Feb 2018 12:09:35 +0100 Subject: [PATCH 2/5] Revert "Fix for Python 2 on Windows, see #1037." This reverts commit b38d31b99d46815967b56fa81468f4ec12b252ee. --- AUTHORS.txt | 1 - jedi/evaluate/compiled/subprocess/__init__.py | 2 -- 2 files changed, 3 deletions(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index bce27b07..691795dd 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -47,6 +47,5 @@ Simon Ruggier (@sruggier) Robin Roth (@robinro) Malte Plath (@langsamer) Anton Zub (@zabulazza) -Tobias Rzepka (@TobiasRzepka) Note: (@user) means a github user name. diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index e4d5a304..a8de8792 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -138,13 +138,11 @@ class _CompiledSubprocess(object): _MAIN_PATH, os.path.dirname(os.path.dirname(parso_path)) ) - universal_newlines = not is_py3 return subprocess.Popen( args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, # stderr=subprocess.PIPE - universal_newlines=universal_newlines ) def run(self, evaluator, function, args=(), kwargs={}): From 99130e7664abd9d1fbc67d098b5399d87cf0758d Mon Sep 17 00:00:00 2001 From: "T.Rzepka" Date: Sat, 17 Feb 2018 13:49:10 +0100 Subject: [PATCH 3/5] Fix for Python 2 and 3 on Windows, see #1037. --- jedi/_compatibility.py | 35 +++++++++++++++---- jedi/evaluate/compiled/subprocess/__init__.py | 2 -- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 73a981a1..52f23b25 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -2,6 +2,8 @@ To ensure compatibility from Python ``2.7`` - ``3.x``, a module has been created. Clearly there is huge need to use conforming syntax. """ +import binascii +import errno import sys import imp import os @@ -434,14 +436,35 @@ if sys.version_info[:2] == (3, 3): _PICKLE_PROTOCOL = 2 - +is_windows = sys.platform == 'win32' def pickle_load(file): - if is_py3: - return pickle.load(file, encoding='bytes') + if is_windows: + try: + data = file.readline() + data = binascii.unhexlify(data.strip()) + if is_py3: + return pickle.loads(data, encoding='bytes') + else: + return pickle.loads(data) + except OSError: + raise EOFError() else: - return pickle.load(file) - + if is_py3: + return pickle.load(file, encoding='bytes') + else: + return pickle.load(file) def pickle_dump(data, file): - pickle.dump(data, file, protocol=_PICKLE_PROTOCOL) + if is_windows: + try: + data = pickle.dumps(data, protocol=_PICKLE_PROTOCOL) + data = binascii.hexlify(data) + file.write(data) + file.write(b'\n') + file.flush() + except OSError: + raise IOError(errno.EPIPE) + else: + pickle.dump(data, file, protocol=_PICKLE_PROTOCOL) + file.flush() diff --git a/jedi/evaluate/compiled/subprocess/__init__.py b/jedi/evaluate/compiled/subprocess/__init__.py index a8de8792..7a87d552 100644 --- a/jedi/evaluate/compiled/subprocess/__init__.py +++ b/jedi/evaluate/compiled/subprocess/__init__.py @@ -187,7 +187,6 @@ class _CompiledSubprocess(object): data = evaluator_id, function, args, kwargs try: pickle_dump(data, self._process.stdin) - self._process.stdin.flush() except (socket.error, IOError) as e: # Once Python2 will be removed we can just use `BrokenPipeError`. if e.errno != errno.EPIPE: @@ -283,7 +282,6 @@ class Listener(object): result = True, traceback.format_exc(), e pickle_dump(result, file=stdout) - stdout.flush() class AccessHandle(object): From 29be40ae3fe61bd2ba50b4f85628c4ba564fad45 Mon Sep 17 00:00:00 2001 From: "T.Rzepka" Date: Sat, 17 Feb 2018 13:52:59 +0100 Subject: [PATCH 4/5] Add author's name to AUTHORS.txt --- AUTHORS.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/AUTHORS.txt b/AUTHORS.txt index abda5916..6d22c03d 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -1,4 +1,4 @@ -Main Authors +Main Authors ============ David Halter (@davidhalter) @@ -48,5 +48,6 @@ Robin Roth (@robinro) Malte Plath (@langsamer) Anton Zub (@zabulazza) Maksim Novikov (@m-novikov) +Tobias Rzepka (@TobiasRzepka) Note: (@user) means a github user name. From e869e700c794c263a9c60d9dc684c83a0e9277ae Mon Sep 17 00:00:00 2001 From: "T.Rzepka" Date: Tue, 20 Feb 2018 21:41:49 +0100 Subject: [PATCH 5/5] Documented the misbehavior of Windows pipes in combination with Python. --- jedi/_compatibility.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/jedi/_compatibility.py b/jedi/_compatibility.py index 52f23b25..f9b4e36c 100644 --- a/jedi/_compatibility.py +++ b/jedi/_compatibility.py @@ -438,6 +438,13 @@ if sys.version_info[:2] == (3, 3): _PICKLE_PROTOCOL = 2 is_windows = sys.platform == 'win32' +# The Windows shell on Python 2 consumes all control characters (below 32) and expand on +# all Python versions \n to \r\n. +# pickle starting from protocol version 1 uses binary data, which could not be escaped by +# any normal unicode encoder. Therefore, the only bytes encoder which doesn't produce +# control characters is binascii.hexlify. + + def pickle_load(file): if is_windows: try: @@ -447,6 +454,8 @@ def pickle_load(file): return pickle.loads(data, encoding='bytes') else: return pickle.loads(data) + # Python on Windows don't throw EOF errors for pipes. So reraise them with + # the correct type, which is cought upwards. except OSError: raise EOFError() else: @@ -455,6 +464,7 @@ def pickle_load(file): else: return pickle.load(file) + def pickle_dump(data, file): if is_windows: try: @@ -462,7 +472,11 @@ def pickle_dump(data, file): data = binascii.hexlify(data) file.write(data) file.write(b'\n') + # On Python 3.3 flush throws sometimes an error even if the two file writes + # should done it already before. This could be also computer / speed depending. file.flush() + # Python on Windows don't throw EPIPE errors for pipes. So reraise them with + # the correct type and error number. except OSError: raise IOError(errno.EPIPE) else: