VimPlug/jedi-fork
1
0
Fork 0
forked from VimPlug/jedi
Code Pull Requests Activity

Temporary parser implementation. Now we're pretty much done with pr.Name.

Browse Source
This commit is contained in:
Dave Halter
2014-09-22 17:05:23 +02:00
parent 6bd7ef56f1
commit 04cf742973
2 changed files with 55 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
jedi/parser/__init__.py
View File

@@ -101,16 +101,17 @@ class Parser(object):
self.module.used_names[tok_name] = set([simple]) self.module.used_names[tok_name] = set([simple])
self.module.temp_used_names = [] self.module.temp_used_names = []
def _parse_dot_name(self, pre_used_token=None): def _parse_dotted_name(self, pre_used_token=None):
""" """
The dot name parser parses a name, variable or function and returns The dot name parser parses a name, variable or function and returns
their names. their names.
Just used for parsing imports.
:return: tuple of Name, next_token :return: tuple of Name, next_token
""" """
def append(el): def append(tok):
names.append(el) names.append(pr.NamePart(self.module, tok.string, None, tok.start_pos))
self.module.temp_used_names.append(el[0]) self.module.temp_used_names.append(tok.string)
names = [] names = []
tok = next(self._gen) if pre_used_token is None else pre_used_token tok = next(self._gen) if pre_used_token is None else pre_used_token
@@ -118,26 +119,23 @@ class Parser(object):
if tok.type != tokenize.NAME and tok.string != '*': if tok.type != tokenize.NAME and tok.string != '*':
return None, tok return None, tok
first_pos = tok.start_pos append(tok)
append((tok.string, first_pos))
while True: while True:
end_pos = tok.end_pos
tok = next(self._gen) tok = next(self._gen)
if tok.string != '.': if tok.string != '.':
break break
tok = next(self._gen) tok = next(self._gen)
if tok.type != tokenize.NAME: if tok.type != tokenize.NAME:
break break
append((tok.string, tok.start_pos)) append(tok)
n = pr.Name(self.module, names, first_pos, end_pos) if names else None return names, tok
return n, tok
def _parse_name(self, pre_used_token=None): def _parse_name(self, pre_used_token=None):
tok = next(self._gen) if pre_used_token is None else pre_used_token tok = next(self._gen) if pre_used_token is None else pre_used_token
if tok.type != tokenize.NAME: if tok.type != tokenize.NAME:
return None return None, tok
return pr.NamePart(self.module, tok.string, None, tok.start_pos) return pr.NamePart(self.module, tok.string, None, tok.start_pos), next(self._gen)
def _parse_import_list(self): def _parse_import_list(self):
""" """
@@ -167,13 +165,13 @@ class Parser(object):
tok = next(self._gen) tok = next(self._gen)
if brackets and tok.type == tokenize.NEWLINE: if brackets and tok.type == tokenize.NEWLINE:
tok = next(self._gen) tok = next(self._gen)
i, tok = self._parse_dot_name(tok) names, tok = self._parse_dotted_name(tok)
if not i: if not names:
defunct = True defunct = True
name2 = None alias = None
if tok.string == 'as': if tok.string == 'as':
name2, tok = self._parse_dot_name() alias, tok = self._parse_name()
imports.append((i, name2, defunct)) imports.append((names, alias, defunct))
while tok.string not in continue_kw: while tok.string not in continue_kw:
tok = next(self._gen) tok = next(self._gen)
if not (tok.string == "," or brackets and tok.type == tokenize.NEWLINE): if not (tok.string == "," or brackets and tok.type == tokenize.NEWLINE):
@@ -230,9 +228,8 @@ class Parser(object):
if tok.type != tokenize.NAME: if tok.type != tokenize.NAME:
return None return None
fname = self._parse_name(tok) fname, tok = self._parse_name(tok)
tok = next(self._gen)
if tok.string != '(': if tok.string != '(':
return None return None
params = self._parse_parentheses(is_class=False) params = self._parse_parentheses(is_class=False)
@@ -269,10 +266,9 @@ class Parser(object):
cname.start_pos[0], tokenize.tok_name[cname.type], cname.string) cname.start_pos[0], tokenize.tok_name[cname.type], cname.string)
return None return None
cname = self._parse_name(cname) cname, _next = self._parse_name(cname)
superclasses = [] superclasses = []
_next = next(self._gen)
if _next.string == '(': if _next.string == '(':
superclasses = self._parse_parentheses(is_class=True) superclasses = self._parse_parentheses(is_class=True)
_next = next(self._gen) _next = next(self._gen)
@@ -346,7 +342,7 @@ class Parser(object):
if tok.string == 'as': if tok.string == 'as':
tok = next(self._gen) tok = next(self._gen)
if tok.type == tokenize.NAME: if tok.type == tokenize.NAME:
n, tok = self._parse_dot_name(self._gen.current) n, tok = self._parse_name(self._gen.current)
if n: if n:
set_vars.append(n) set_vars.append(n)
as_names.append(n) as_names.append(n)
@@ -358,11 +354,6 @@ class Parser(object):
elif in_lambda_param and tok.string == ':': elif in_lambda_param and tok.string == ':':
in_lambda_param = False in_lambda_param = False
elif tok.type == tokenize.NAME and not is_kw: elif tok.type == tokenize.NAME and not is_kw:
n, tok = self._parse_dot_name(self._gen.current)
# removed last entry, because we add Name
tok_list.pop()
if n:
tok_list.append(n)
continue continue
elif tok.string in opening_brackets: elif tok.string in opening_brackets:
level += 1 level += 1
@@ -464,10 +455,10 @@ class Parser(object):
# import stuff # import stuff
elif tok_str == 'import': elif tok_str == 'import':
imports = self._parse_import_list() imports = self._parse_import_list()
for count, (m, alias, defunct) in enumerate(imports): for count, (names, alias, defunct) in enumerate(imports):
e = (alias or m or self._gen.previous).end_pos e = (alias or m or self._gen.previous).end_pos
end_pos = self._gen.previous.end_pos if count + 1 == len(imports) else e end_pos = self._gen.previous.end_pos if count + 1 == len(imports) else e
i = pr.Import(self.module, first_pos, end_pos, m, i = pr.Import(self.module, first_pos, end_pos, names,
alias, defunct=defunct) alias, defunct=defunct)
self._check_user_stmt(i) self._check_user_stmt(i)
self._scope.add_import(i) self._scope.add_import(i)
@@ -486,26 +477,26 @@ class Parser(object):
break break
relative_count += 1 relative_count += 1
# the from import # the from import
mod, tok = self._parse_dot_name(self._gen.current) from_names, tok = self._parse_dotted_name(self._gen.current)
tok_str = tok.string tok_str = tok.string
if str(mod) == 'import' and relative_count: if len(from_names) == 1 and str(from_names[0]) == 'import' and relative_count:
self._gen.push_last_back() self._gen.push_last_back()
tok_str = 'import' tok_str = 'import'
mod = None names = []
if not mod and not relative_count or tok_str != "import": if not names and not relative_count or tok_str != "import":
debug.warning("from: syntax error@%s", tok.start_pos[0]) debug.warning("from: syntax error@%s", tok.start_pos[0])
defunct = True defunct = True
if tok_str != 'import': if tok_str != 'import':
self._gen.push_last_back() self._gen.push_last_back()
names = self._parse_import_list() names = self._parse_import_list()
for count, (name, alias, defunct2) in enumerate(names): for count, (names, alias, defunct2) in enumerate(names):
star = name is not None and unicode(name.names[0]) == '*' star = names and unicode(names) == '*'
if star: if star:
name = None names = []
e = (alias or name or self._gen.previous).end_pos e = (alias or names and names[-1] or self._gen.previous).end_pos
end_pos = self._gen.previous.end_pos if count + 1 == len(names) else e end_pos = self._gen.previous.end_pos if count + 1 == len(names) else e
i = pr.Import(self.module, first_pos, end_pos, name, i = pr.Import(self.module, first_pos, end_pos, names,
alias, mod, star, relative_count, alias, from_names, star, relative_count,
defunct=defunct or defunct2) defunct=defunct or defunct2)
self._check_user_stmt(i) self._check_user_stmt(i)
self._scope.add_import(i) self._scope.add_import(i)
@@ -540,7 +531,7 @@ class Parser(object):
if command == 'except' and tok.string == ',': if command == 'except' and tok.string == ',':
# the except statement defines a var # the except statement defines a var
# this is only true for python 2 # this is only true for python 2
n, tok = self._parse_dot_name() n, tok = self._parse_name()
if n: if n:
n.parent = statement n.parent = statement
statement.as_names.append(n) statement.as_names.append(n)

52
jedi/parser/representation.py
View File

@@ -768,25 +768,25 @@ class Import(Simple):
:param start_pos: Position (line, column) of the Import. :param start_pos: Position (line, column) of the Import.
:type start_pos: tuple(int, int) :type start_pos: tuple(int, int)
:param namespace: The import, can be empty if a star is given :param namespace_names: The import, can be empty if a star is given
:type namespace: Name :type namespace_names: Name
:param alias: The alias of a namespace(valid in the current namespace). :param alias: The alias of a namespace(valid in the current namespace).
:type alias: Name :type alias: Name
:param from_ns: Like the namespace, can be equally used. :param from_names: Like the namespace, can be equally used.
:type from_ns: Name :type from_names: Name
:param star: If a star is used -> from time import *. :param star: If a star is used -> from time import *.
:type star: bool :type star: bool
:param defunct: An Import is valid or not. :param defunct: An Import is valid or not.
:type defunct: bool :type defunct: bool
""" """
def __init__(self, module, start_pos, end_pos, namespace, alias=None, def __init__(self, module, start_pos, end_pos, namespace_names, alias=None,
from_ns=None, star=False, relative_count=0, defunct=False): from_names=None, star=False, relative_count=0, defunct=False):
super(Import, self).__init__(module, start_pos, end_pos) super(Import, self).__init__(module, start_pos, end_pos)
self.namespace = namespace self.namespace_names = namespace_names
self.alias = alias self.alias = alias
self.from_ns = from_ns self.from_names = from_names
for n in namespace, alias, from_ns: for n in namespace_names, alias, from_names:
if n: if n:
n.parent = self.use_as_parent n.parent = self.use_as_parent
@@ -797,20 +797,19 @@ class Import(Simple):
def get_code(self, new_line=True): def get_code(self, new_line=True):
# in case one of the names is None # in case one of the names is None
alias = self.alias or '' alias = self.alias or ''
namespace = self.namespace or ''
from_ns = self.from_ns or ''
namespace = '.'.join(self.namespace_names)
if self.alias: if self.alias:
ns_str = "%s as %s" % (namespace, alias) ns_str = "%s as %s" % ('.'.join(namespace), alias)
else: else:
ns_str = unicode(namespace) ns_str = namespace
nl = '\n' if new_line else '' nl = '\n' if new_line else ''
if self.from_ns or self.relative_count: if self.from_names or self.relative_count:
if self.star: if self.star:
ns_str = '*' ns_str = '*'
dots = '.' * self.relative_count dots = '.' * self.relative_count
return "from %s%s import %s%s" % (dots, from_ns, ns_str, nl) return "from %s%s import %s%s" % (dots, '.'.join(self.from_names), ns_str, nl)
else: else:
return "import %s%s" % (ns_str, nl) return "import %s%s" % (ns_str, nl)
@@ -821,22 +820,19 @@ class Import(Simple):
return [self] return [self]
if self.alias: if self.alias:
return [self.alias] return [self.alias]
if len(self.namespace) > 1: if len(self.namespace_names) > 1:
o = self.namespace return self.namespace_names[0]
n = Name(self._sub_module, [(unicode(o.names[0]), o.start_pos)],
o.start_pos, o.end_pos, parent=o.parent)
return [n]
else: else:
return [self.namespace] return [self.namespace_names]
def get_all_import_names(self): def get_all_import_names(self):
n = [] n = []
if self.from_ns: if self.from_names:
n += self.from_ns.names n += self.from_names
if self.namespace: if self.namespace_names:
n += self.namespace.names n += self.namespace_names
if self.alias: if self.alias is not None:
n += self.alias.names n.append(self.alias)
return n return n
@property @property
@@ -854,7 +850,7 @@ class Import(Simple):
import foo.bar import foo.bar
""" """
return not self.alias and not self.from_ns and self.namespace is not None \ return not self.alias and not self.from_names and self.namespace is not None \
and len(self.namespace.names) > 1 and len(self.namespace.names) > 1