From 38474061cf8cbb8dd0cedab60919934201b02597 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 29 Jun 2018 09:54:57 +0200 Subject: [PATCH 1/6] Make jedi work with the next parso release --- jedi/api/classes.py | 5 ++-- jedi/api/completion.py | 44 +++++++++++++++++--------------- jedi/api/helpers.py | 3 +-- test/test_api/test_completion.py | 4 +++ tox.ini | 2 +- 5 files changed, 32 insertions(+), 26 deletions(-) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index cfcbf8db..633e8e3d 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -404,8 +404,9 @@ class Completion(BaseDefinition): append = '(' if self._name.api_type == 'param' and self._stack is not None: - node_names = list(self._stack.get_node_names(self._evaluator.grammar._pgen_grammar)) - if 'trailer' in node_names and 'argument' not in node_names: + nonterminals = list(self._stack._list_nonterminals()) + if 'trailer' in nonterminals and 'argument' not in nonterminals: + # TODO this doesn't work for nested calls. append += '=' name = self._name.string_name diff --git a/jedi/api/completion.py b/jedi/api/completion.py index 020a096c..4b078d4c 100644 --- a/jedi/api/completion.py +++ b/jedi/api/completion.py @@ -1,4 +1,4 @@ -from parso.python import token +from parso.python.token import PythonTokenTypes from parso.python import tree from parso.tree import search_ancestor, Leaf @@ -122,11 +122,11 @@ class Completion: grammar = self._evaluator.grammar try: - self.stack = helpers.get_stack_at_position( + self.stack = stack = helpers.get_stack_at_position( grammar, self._code_lines, self._module_node, self._position ) except helpers.OnErrorLeaf as e: - self.stack = None + self.stack = stack = None if e.error_leaf.value == '.': # After ErrorLeaf's that are dots, we will not do any # completions since this probably just confuses the user. @@ -135,10 +135,10 @@ class Completion: return self._global_completions() - allowed_keywords, allowed_tokens = \ - helpers.get_possible_completion_types(grammar._pgen_grammar, self.stack) + allowed_transitions = \ + list(stack._allowed_transition_names_and_token_types()) - if 'if' in allowed_keywords: + if 'if' in allowed_transitions: leaf = self._module_node.get_leaf_for_position(self._position, include_prefixes=True) previous_leaf = leaf.get_previous_leaf() @@ -164,50 +164,52 @@ class Completion: # Compare indents if stmt.start_pos[1] == indent: if type_ == 'if_stmt': - allowed_keywords += ['elif', 'else'] + allowed_transitions += ['elif', 'else'] elif type_ == 'try_stmt': - allowed_keywords += ['except', 'finally', 'else'] + allowed_transitions += ['except', 'finally', 'else'] elif type_ == 'for_stmt': - allowed_keywords.append('else') + allowed_transitions.append('else') - completion_names = list(self._get_keyword_completion_names(allowed_keywords)) + completion_names = list(self._get_keyword_completion_names(allowed_transitions)) - if token.NAME in allowed_tokens or token.INDENT in allowed_tokens: + if any(t in allowed_transitions for t in (PythonTokenTypes.NAME, + PythonTokenTypes.INDENT)): # This means that we actually have to do type inference. - symbol_names = list(self.stack.get_node_names(grammar._pgen_grammar)) + nonterminals = stack._list_nonterminals() - nodes = list(self.stack.get_nodes()) + nodes = stack._list_all_nodes() if nodes and nodes[-1] in ('as', 'def', 'class'): # No completions for ``with x as foo`` and ``import x as foo``. # Also true for defining names as a class or function. return list(self._get_class_context_completions(is_function=True)) - elif "import_stmt" in symbol_names: - level, names = self._parse_dotted_names(nodes, "import_from" in symbol_names) + elif "import_stmt" in nonterminals: + level, names = self._parse_dotted_names(nodes, "import_from" in nonterminals) - only_modules = not ("import_from" in symbol_names and 'import' in nodes) + only_modules = not ("import_from" in nonterminals and 'import' in nodes) completion_names += self._get_importer_names( names, level, only_modules=only_modules, ) - elif symbol_names[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.': + elif nonterminals[-1] in ('trailer', 'dotted_name') and nodes[-1] == '.': dot = self._module_node.get_leaf_for_position(self._position) completion_names += self._trailer_completions(dot.get_previous_leaf()) else: completion_names += self._global_completions() completion_names += self._get_class_context_completions(is_function=False) - if 'trailer' in symbol_names: + if 'trailer' in nonterminals: call_signatures = self._call_signatures_method() completion_names += get_call_signature_param_names(call_signatures) return completion_names - def _get_keyword_completion_names(self, keywords_): - for k in keywords_: - yield keywords.KeywordName(self._evaluator, k) + def _get_keyword_completion_names(self, allowed_transitions): + for k in allowed_transitions: + if isinstance(k, str) and k.isalpha(): + yield keywords.KeywordName(self._evaluator, k) def _global_completions(self): context = get_user_scope(self._module_context, self._position) diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 221fc4df..6fcca339 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -127,7 +127,7 @@ def get_stack_at_position(grammar, code_lines, module_node, pos): try: p.parse(tokens=tokenize_without_endmarker(code)) except EndMarkerReached: - return Stack(p.pgen_parser.stack) + return p.stack raise SystemError("This really shouldn't happen. There's a bug in Jedi.") @@ -153,7 +153,6 @@ def get_possible_completion_types(pgen_grammar, stack): t, v = pgen_grammar.labels[label_index] assert t >= 256 # See if it's a symbol and if we're in its first set - inversed_keywords itsdfa = pgen_grammar.dfas[t] itsstates, itsfirst = itsdfa for first_label_index in itsfirst.keys(): diff --git a/test/test_api/test_completion.py b/test/test_api/test_completion.py index 50c452b7..a40095b9 100644 --- a/test/test_api/test_completion.py +++ b/test/test_api/test_completion.py @@ -137,3 +137,7 @@ def test_async(Script, environment): names = [c.name for c in comps] assert 'foo' in names assert 'hey' in names + + +def test_with_stmt_error_recovery(Script): + assert Script('with open('') as foo: foo.\na', line=1).completions() diff --git a/tox.ini b/tox.ini index bf909253..96483079 100644 --- a/tox.ini +++ b/tox.ini @@ -24,7 +24,7 @@ setenv = env37: JEDI_TEST_ENVIRONMENT=37 commands = # Overwrite the parso version (only used sometimes). -# pip install git+https://github.com/davidhalter/parso.git + pip install git+https://github.com/davidhalter/parso.git py.test {posargs:jedi test} [testenv:py27] deps = From c208d37ac481785e60f661ebf74c2962039e0423 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 29 Jun 2018 09:56:56 +0200 Subject: [PATCH 2/6] Remove code that is no longer used, because parso was refactored. --- jedi/api/helpers.py | 51 --------------------------------------------- 1 file changed, 51 deletions(-) diff --git a/jedi/api/helpers.py b/jedi/api/helpers.py index 6fcca339..cb764cc6 100644 --- a/jedi/api/helpers.py +++ b/jedi/api/helpers.py @@ -12,7 +12,6 @@ from jedi._compatibility import u from jedi.evaluate.syntax_tree import eval_atom from jedi.evaluate.helpers import evaluate_call_of_leaf from jedi.evaluate.compiled import get_string_context_set -from jedi.evaluate.base_context import ContextSet from jedi.cache import call_signature_time_cache @@ -131,56 +130,6 @@ def get_stack_at_position(grammar, code_lines, module_node, pos): raise SystemError("This really shouldn't happen. There's a bug in Jedi.") -class Stack(list): - def get_node_names(self, grammar): - for dfa, state, (node_number, nodes) in self: - yield grammar.number2symbol[node_number] - - def get_nodes(self): - for dfa, state, (node_number, nodes) in self: - for node in nodes: - yield node - - -def get_possible_completion_types(pgen_grammar, stack): - def add_results(label_index): - try: - grammar_labels.append(inversed_tokens[label_index]) - except KeyError: - try: - keywords.append(inversed_keywords[label_index]) - except KeyError: - t, v = pgen_grammar.labels[label_index] - assert t >= 256 - # See if it's a symbol and if we're in its first set - itsdfa = pgen_grammar.dfas[t] - itsstates, itsfirst = itsdfa - for first_label_index in itsfirst.keys(): - add_results(first_label_index) - - inversed_keywords = dict((v, k) for k, v in pgen_grammar.keywords.items()) - inversed_tokens = dict((v, k) for k, v in pgen_grammar.tokens.items()) - - keywords = [] - grammar_labels = [] - - def scan_stack(index): - dfa, state, node = stack[index] - states, first = dfa - arcs = states[state] - - for label_index, new_state in arcs: - if label_index == 0: - # An accepting state, check the stack below. - scan_stack(index - 1) - else: - add_results(label_index) - - scan_stack(-1) - - return keywords, grammar_labels - - def evaluate_goto_definition(evaluator, context, leaf): if leaf.type == 'name': # In case of a name we can just use goto_definition which does all the From 68974aee5804aff5966869e3630c164b210cfcf1 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 29 Jun 2018 10:04:03 +0200 Subject: [PATCH 3/6] Don't use internal parso APIs if possible --- jedi/api/classes.py | 2 +- jedi/api/completion.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/jedi/api/classes.py b/jedi/api/classes.py index 633e8e3d..4168f6e9 100644 --- a/jedi/api/classes.py +++ b/jedi/api/classes.py @@ -404,7 +404,7 @@ class Completion(BaseDefinition): append = '(' if self._name.api_type == 'param' and self._stack is not None: - nonterminals = list(self._stack._list_nonterminals()) + nonterminals = [stack_node.nonterminal for stack_node in self._stack] if 'trailer' in nonterminals and 'argument' not in nonterminals: # TODO this doesn't work for nested calls. append += '=' diff --git a/jedi/api/completion.py b/jedi/api/completion.py index 4b078d4c..358d726b 100644 --- a/jedi/api/completion.py +++ b/jedi/api/completion.py @@ -176,9 +176,9 @@ class Completion: PythonTokenTypes.INDENT)): # This means that we actually have to do type inference. - nonterminals = stack._list_nonterminals() + nonterminals = [stack_node.nonterminal for stack_node in stack] - nodes = stack._list_all_nodes() + nodes = [node for stack_node in stack for node in stack_node.nodes] if nodes and nodes[-1] in ('as', 'def', 'class'): # No completions for ``with x as foo`` and ``import x as foo``. From 1e7662c3e1cfcff60a5e8277b1ed126668f248ed Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 29 Jun 2018 18:10:41 +0200 Subject: [PATCH 4/6] Prepare release of 0.12.1 --- CHANGELOG.rst | 6 ++++++ jedi/__init__.py | 2 +- requirements.txt | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index a02b0ed5..8192cd68 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -3,6 +3,12 @@ Changelog --------- +0.12.1 (2018-06-30) ++++++++++++++++++++ + +- This release forces you to upgrade parso. If you don't, nothing will work + anymore. Otherwise changes should be limited to bug fixes. + 0.12.0 (2018-04-15) +++++++++++++++++++ diff --git a/jedi/__init__.py b/jedi/__init__.py index ff2de906..30b19216 100644 --- a/jedi/__init__.py +++ b/jedi/__init__.py @@ -36,7 +36,7 @@ As you see Jedi is pretty simple and allows you to concentrate on writing a good text editor, while still having very good IDE features for Python. """ -__version__ = '0.12.0' +__version__ = '0.12.1' from jedi.api import Script, Interpreter, set_debug_function, \ preload_module, names diff --git a/requirements.txt b/requirements.txt index 4bdf1b74..bde6897f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -parso>=0.2.0 +parso>=0.3.0 From e0e2be3027d9379384e7feffef34f084e786cbfa Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Fri, 29 Jun 2018 18:17:29 +0200 Subject: [PATCH 5/6] Add a better comment about why people need to upgrade parso --- CHANGELOG.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8192cd68..6c1794e5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,9 @@ Changelog +++++++++++++++++++ - This release forces you to upgrade parso. If you don't, nothing will work - anymore. Otherwise changes should be limited to bug fixes. + anymore. Otherwise changes should be limited to bug fixes. Unfortunately Jedi + still uses a few internals of parso that make it hard to keep compatibility + over multiple releases. Parso >=0.3.0 is going to be needed. 0.12.0 (2018-04-15) +++++++++++++++++++ From 58141f1e1e4d1cc6a589de28dc5f067f1ddc60e2 Mon Sep 17 00:00:00 2001 From: Dave Halter Date: Sat, 30 Jun 2018 14:14:52 +0200 Subject: [PATCH 6/6] Don't use requirements for now, and use the git version instead in tox --- tox.ini | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 96483079..9ecda0fd 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,9 @@ deps = docopt # coloroma for colored debug output colorama - -rrequirements.txt +# Overwrite the parso version (only used sometimes). + git+https://github.com/davidhalter/parso.git +# -rrequirements.txt passenv = JEDI_TEST_ENVIRONMENT setenv = # https://github.com/tomchristie/django-rest-framework/issues/1957 @@ -23,8 +25,6 @@ setenv = env36: JEDI_TEST_ENVIRONMENT=36 env37: JEDI_TEST_ENVIRONMENT=37 commands = -# Overwrite the parso version (only used sometimes). - pip install git+https://github.com/davidhalter/parso.git py.test {posargs:jedi test} [testenv:py27] deps =