diff --git a/ISSUES.md b/ISSUES.md index 13db25b..5b95107 100644 --- a/ISSUES.md +++ b/ISSUES.md @@ -2,7 +2,7 @@ ## Detection of catching some exceptions -Docstring for this method will conain 'Raises Exception': +Docstring for this method will contain 'Raises Exception': ~~~{python} def foo(): @@ -28,4 +28,4 @@ Comment return x ~~~ -This will probably result in an error, though the code is valid. \ No newline at end of file +This will probably result in an error, though the code is valid. diff --git a/README.md b/README.md index 2c71cfb..93b20a0 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # vim-python-docstring -This is a plugin to Vim and NeoVim for creating of docstrings. - -**New**: Support for type hints and async functions. +This is a plugin to Vim and NeoVim for the creation of Python docstrings. ## What it does Docstrings for methods will contain a **list of parameters and their type hints**, **list of raised exceptions** and whether the method **yields** or **raises**. @@ -45,13 +43,13 @@ The plugin uses these commands: ## Options: There are things you can set. -### The `g:python_indent` option +### The `g:vpd_indent` option String which you use to indent your code. Default: `' '` (4 spaces). ~~~{viml} -let g:python_indent = ' ' +let g:vpd_indent = ' ' ~~~ ### The `g:python_style` option diff --git a/autoload/vimpythondocstring.vim b/autoload/vimpythondocstring.vim index 180962f..189170c 100644 --- a/autoload/vimpythondocstring.vim +++ b/autoload/vimpythondocstring.vim @@ -12,14 +12,32 @@ sys.path[0:0] = deps import pydocstring EOF +function! s:handle_error(exception) + echohl ErrorMsg + echo join(map(split(a:exception, ":")[2:], 'trim(v:val)'), " : ") + echohl None +endfunction + function! vimpythondocstring#Full() - python3 pydocstring.Docstring().full_docstring() + try + python3 pydocstring.Docstring().full_docstring() + catch + call s:handle_error(v:exception) + endtry endfunction function! vimpythondocstring#FullTypes() - python3 pydocstring.Docstring().full_docstring(print_hints=True) + try + python3 pydocstring.Docstring().full_docstring(print_hints=True) + catch + call s:handle_error(v:exception) + endtry endfunction function! vimpythondocstring#Oneline() - python3 pydocstring.Docstring().oneline_docstring() + try + python3 pydocstring.Docstring().oneline_docstring() + catch + call s:handle_error(v:exception) + endtry endfunction diff --git a/python/asthelper.py b/python/asthelper.py index bc9a595..7c5b6a4 100644 --- a/python/asthelper.py +++ b/python/asthelper.py @@ -1,8 +1,9 @@ import ast +import sys +from itertools import chain class RaiseNameCollector(ast.NodeVisitor): - def __init__(self): self.data = set() super().__init__() @@ -12,29 +13,27 @@ def visit_Call(self, node): class AttributeCollector(ast.NodeVisitor): - def __init__(self, instance_name): self.instance_name = instance_name - self.data = set() + self.data = {} super().__init__() def visit_Attribute(self, node): if isinstance(node.value, ast.Name): if node.value.id == self.instance_name: - self.data.add(node.attr) + self.data[node.attr] = None else: self.generic_visit(node) class ClassInstanceNameExtractor(ast.NodeVisitor): - def __init__(self): - self.instance_name = 'self' # default + self.instance_name = "self" # default self.set = False super().__init__() def visit_FunctionDef(self, node): - if node.name == '__init__': + if node.name == "__init__": self.instance_name = node.args.args[0].arg self.set = True elif not self.set: @@ -48,18 +47,23 @@ def generic_visit(self, node): class ClassVisitor(ast.NodeVisitor): def __init__(self, instance_name): super().__init__() - self.attributes = set() + self.attributes = {} self.instance_name = instance_name def visit_Assign(self, node): ac = AttributeCollector(self.instance_name) for target in node.targets: - ac.visit(node) + ac.visit(target) + self.attributes |= ac.data + + def visit_AnnAssign(self, node): + ac = AttributeCollector(self.instance_name) + ac.visit(node.target) self.attributes |= ac.data class MethodVisitor(ast.NodeVisitor): - """ Gathers information about a method + """Gathers information about a method Attributes: arguments: arguments of the method @@ -69,6 +73,7 @@ class MethodVisitor(ast.NodeVisitor): yields: True is method yields """ + def __init__(self, parent=True): self.parent = parent self.arguments = [] @@ -83,12 +88,22 @@ def _handle_functions(self, node): self.raises |= new_visitor.raises if self.parent: - for arg in node.args.args: + for arg in chain(node.args.args, node.args.kwonlyargs): type_hint = None if arg.annotation is not None: - type_hint = ast.unparse(arg.annotation) - self.arguments.append({'arg': arg.arg, 'type': type_hint}) - if len(self.arguments) > 0 and (self.arguments[0]['arg'] == 'self' or self.arguments[0]['arg'] == 'cls'): + # ast.unparse doesn't work for python <= 3.8 + if sys.version_info[0] == 3 and sys.version_info[1] <= 8: + from unparse import Unparser + from io import StringIO + v = StringIO() + Unparser(arg.annotation, file=v) + type_hint = v.getvalue() + else: + type_hint = ast.unparse(arg.annotation) + self.arguments.append({"arg": arg.arg, "type": type_hint}) + if len(self.arguments) > 0 and ( + self.arguments[0]["arg"] == "self" or self.arguments[0]["arg"] == "cls" + ): self.arguments.pop(0) self.returns = new_visitor.returns diff --git a/python/pydocstring.py b/python/pydocstring.py index a60579e..9b4c0c8 100644 --- a/python/pydocstring.py +++ b/python/pydocstring.py @@ -1,29 +1,30 @@ #!/usr/bin/env python3 -from string import Template -import re -import os -import ast import abc +import ast +import os +import re +from string import Template import ibis - +from asthelper import ClassInstanceNameExtractor, ClassVisitor, MethodVisitor from utils import * from vimenv import * -from asthelper import ClassVisitor, MethodVisitor, ClassInstanceNameExtractor class InvalidSyntax(Exception): - """ Raise when the syntax of processed object is invalid. """ + """Raise when the syntax of processed object is invalid.""" + pass class DocstringUnavailable(Exception): - """ Raise when trying to process object to which there is no docstring. """ + """Raise when trying to process object to which there is no docstring.""" + pass class Templater: - """ Class used to template the docstrings + """Class used to template the docstrings Attributes: indent: used indentation @@ -33,36 +34,54 @@ class Templater: """ - def __init__(self, location, indent, style='google'): + def __init__(self, location, indent, style="google"): self.style = style self.indent = indent self.location = location def _docstring_helper(self, obj_indent, docstring): lines = [] - for line in docstring.split('\n'): - if re.match('.', line): + for line in docstring.split("\n"): + if re.match(".", line): line = concat_(obj_indent, self.indent, line) lines.append(line) - return '\n'.join(lines) - - def get_method_docstring(self, method_indent, args, returns, yields, raises, print_hints=False): - with open(os.path.join(self.location, '..', 'styles/{}-{}.txt'.format(self.style, 'method')), 'r') as f: + return "\n".join(lines) + + def get_method_docstring( + self, method_indent, args, returns, yields, raises, print_hints=False + ): + with open( + os.path.join( + self.location, "..", "styles/{}-{}.txt".format(self.style, "method") + ), + "r", + ) as f: self.template = ibis.Template(f.read()) - docstring = self.template.render(indent=self.indent, args=args, hints=print_hints, - raises=raises, returns=returns, yields=yields) + docstring = self.template.render( + indent=self.indent, + args=args, + hints=print_hints, + raises=raises, + returns=returns, + yields=yields, + ) return self._docstring_helper(method_indent, docstring) def get_class_docstring(self, class_indent, attr): - with open(os.path.join(self.location, '..', 'styles/{}-{}.txt'.format(self.style, 'class')), 'r') as f: + with open( + os.path.join( + self.location, "..", "styles/{}-{}.txt".format(self.style, "class") + ), + "r", + ) as f: self.template = ibis.Template(f.read()) docstring = self.template.render(indent=self.indent, attr=attr) return self._docstring_helper(class_indent, docstring) class ObjectWithDocstring(abc.ABC): - """ Represents an object (class, method) with the enviroment in which it is opened + """Represents an object (class, method) with the enviroment in which it is opened Attributes: env: enviroment class @@ -78,7 +97,7 @@ def __init__(self, env, templater): @abc.abstractmethod def write_docstring(self, *args, **kwargs): - """ Method to create a docstring for appropriate object + """Method to create a docstring for appropriate object Writes the docstring to correct lines in `self.env` object. """ @@ -88,27 +107,27 @@ def _get_sig(self): lines = [] lines_it = self.env.lines_following_cursor() sig_line, first_line = next(lines_it) - indent = re.findall(r'^(\s*)', first_line)[0] + indent = re.findall(r"^(\s*)", first_line)[0] lines.append(first_line) - while not self._is_valid(''.join(lines)): + while not self._is_valid("".join(lines)): try: sig_line, line = next(lines_it) except StopIteration as e: - raise InvalidSyntax('Object does not have valid syntax') + raise InvalidSyntax("Object does not have valid syntax") lines.append(line) return sig_line, indent def _object_tree(self): - """ Get the source code of the object under cursor. """ + """Get the source code of the object under cursor.""" lines = [] lines_it = self.env.lines_following_cursor() sig_line, first_line = next(lines_it) lines.append(first_line) - obj_indent = re.findall(r'^(\s*)', first_line)[0] + obj_indent = re.findall(r"^(\s*)", first_line)[0] expected_indent = concat_(obj_indent, self.env.python_indent) valid_sig, _ = self._is_valid(first_line) @@ -119,54 +138,56 @@ def _object_tree(self): except Exception as e: break - if valid_sig and not self._is_correct_indent(lines[-1], line, expected_indent): + if valid_sig and not self._is_correct_indent( + lines[-1], line, expected_indent + ): break lines.append(line) if not valid_sig: - data = ''.join(lines) + data = "".join(lines) valid_sig, _ = self._is_valid(data) sig_line = last_row # remove obj_indent from the beginning of all lines - lines = [re.sub('^'+obj_indent, '', l) for l in lines] + lines = [re.sub("^" + obj_indent, "", l) for l in lines] for i, l in enumerate(reversed(lines)): - if l.strip() == '': + if l.strip() == "": lines.pop() else: break if len(lines) == 1: - lines.append(f'{self.env.python_indent}pass') + lines.append(f"{self.env.python_indent}pass") - data = '\n'.join(lines) + data = "\n".join(lines) try: tree = ast.parse(data) except Exception as e: - raise InvalidSyntax('Object has invalid syntax.') + raise InvalidSyntax("Object has invalid syntax.") return sig_line, obj_indent, tree def _is_correct_indent(self, previous_line, line, expected_indent): - """ Check whether given line has either given indentation (or more) - or does contain only nothing or whitespaces. + """Check whether given line has either given indentation (or more) + or does contain only nothing or whitespaces. """ # Disclaimer: I know this does not check for multiline comments and strings # strings ''' .....''' are a problem !!! - if re.match('^'+expected_indent, line): + if re.match(r"^" + expected_indent, line): return True - elif re.match('^\s*#', line): + elif re.match(r"^\s*#", line): return True - elif re.match('^\s*["\']{3}', line): + elif re.match(r"^\s*[\"']{3}", line): return True - elif re.match('.*\\$', previous_line): + elif re.match(r".*\\$", previous_line): return True - elif re.match('^\s*$', line): + elif re.match(r"^\s*$", line): return True return False def _is_valid(self, lines): - func = concat_(lines.lstrip(), '\n pass') + func = concat_(lines.lstrip(), "\n pass") try: tree = ast.parse(func) return True, tree @@ -174,14 +195,13 @@ def _is_valid(self, lines): return False, None def write_simple_docstring(self): - """ Writes the generated docstring in the enviroment """ + """Writes the generated docstring in the enviroment""" sig_line, indent = self._get_sig() docstring = concat_(indent, self.templater.indent, '""" """') self.env.append_after_line(sig_line, docstring) class MethodController(ObjectWithDocstring): - def __init__(self, env, templater): super().__init__(env, templater) @@ -197,12 +217,12 @@ def write_docstring(self, print_hints=False): sig_line, method_indent, tree = self._object_tree() args, returns, yields, raises = self._process_tree(tree) docstring = self.templater.get_method_docstring( - method_indent, args, returns, yields, raises, print_hints) + method_indent, args, returns, yields, raises, print_hints + ) self.env.append_after_line(sig_line, docstring) class ClassController(ObjectWithDocstring): - def __init__(self, env, templater): super().__init__(env, templater) @@ -211,7 +231,7 @@ def _process_tree(self, tree): x.visit(tree) v = ClassVisitor(x.instance_name) v.visit(tree) - att = list(v.attributes) + att = [attr_name for attr_name in v.attributes] return att def write_docstring(self, *args, **kwargs): @@ -222,7 +242,7 @@ def write_docstring(self, *args, **kwargs): class Docstring: - """ Class used by user to generate docstrings""" + """Class used by user to generate docstrings""" def __init__(self): env = VimEnviroment() @@ -235,32 +255,33 @@ def __init__(self): def _controller_factory(self, env, templater): line = env.current_line - first_word = re.match(r'^\s*(\w+).*', line).groups()[0] - if first_word == 'def': + try: + first_word = re.match(r"^\s*(\w+).*", line).groups()[0] + except Exception: + first_word = None + if first_word == "def": return MethodController(env, templater) - elif first_word == 'class': + elif first_word == "class": return ClassController(env, templater) - elif first_word == 'async': - second_word_catch = re.match(r'^\s*\w+\s+(\w+).*', line) + elif first_word == "async": + second_word_catch = re.match(r"^\s*\w+\s+(\w+).*", line) if second_word_catch: second_word = second_word_catch.groups()[0] - if second_word == 'def': + if second_word == "def": return MethodController(env, templater) - raise DocstringUnavailable( - 'Docstring cannot be created for selected object') + raise DocstringUnavailable("Docstring ERROR: Doctring cannot be created for selected object") def full_docstring(self, print_hints=False): - """ Writes docstring containing arguments, returns, raises, ... """ + """Writes docstring containing arguments, returns, raises, ...""" try: self.obj_controller.write_docstring(print_hints=print_hints) except Exception as e: - print(concat_('Doctring ERROR: ', e)) + raise DocstringUnavailable(concat_("Docstring ERROR: ", e)) def oneline_docstring(self): - """ Writes only a one-line empty docstring """ + """Writes only a one-line empty docstring""" try: self.obj_controller.write_simple_docstring() except Exception as e: - print(concat_('Doctring ERROR: ', e)) - + raise DocstringUnavailable(concat_("Docstring ERROR: ", e)) diff --git a/python/unparse.py b/python/unparse.py new file mode 100644 index 0000000..1225f0d --- /dev/null +++ b/python/unparse.py @@ -0,0 +1,722 @@ +""" +Usage: unparse.py +Taken from https://github.com/python/cpython/blob/3.8/Tools/parser/unparse.py +""" +import sys +import ast +import tokenize +import io +import os + +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + +def interleave(inter, f, seq): + """Call f on each item in seq, calling inter() in between. + """ + seq = iter(seq) + try: + f(next(seq)) + except StopIteration: + pass + else: + for x in seq: + inter() + f(x) + +class Unparser: + """Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarded. """ + + def __init__(self, tree, file = sys.stdout): + """Unparser(tree, file=sys.stdout) -> None. + Print the source for tree to file.""" + self.f = file + self._indent = 0 + self.dispatch(tree) + print("", file=self.f) + self.f.flush() + + def fill(self, text = ""): + "Indent a piece of text, according to the current indentation level" + self.f.write("\n"+" "*self._indent + text) + + def write(self, text): + "Append a piece of text to the current line." + self.f.write(text) + + def enter(self): + "Print ':', and increase the indentation." + self.write(":") + self._indent += 1 + + def leave(self): + "Decrease the indentation level." + self._indent -= 1 + + def dispatch(self, tree): + "Dispatcher function, dispatching tree type T to method _T." + if isinstance(tree, list): + for t in tree: + self.dispatch(t) + return + meth = getattr(self, "_"+tree.__class__.__name__) + meth(tree) + + + ############### Unparsing methods ###################### + # There should be one method per concrete grammar type # + # Constructors should be grouped by sum type. Ideally, # + # this would follow the order in the grammar, but # + # currently doesn't. # + ######################################################## + + def _Module(self, tree): + for stmt in tree.body: + self.dispatch(stmt) + + # stmt + def _Expr(self, tree): + self.fill() + self.dispatch(tree.value) + + def _NamedExpr(self, tree): + self.write("(") + self.dispatch(tree.target) + self.write(" := ") + self.dispatch(tree.value) + self.write(")") + + def _Import(self, t): + self.fill("import ") + interleave(lambda: self.write(", "), self.dispatch, t.names) + + def _ImportFrom(self, t): + self.fill("from ") + self.write("." * t.level) + if t.module: + self.write(t.module) + self.write(" import ") + interleave(lambda: self.write(", "), self.dispatch, t.names) + + def _Assign(self, t): + self.fill() + for target in t.targets: + self.dispatch(target) + self.write(" = ") + self.dispatch(t.value) + + def _AugAssign(self, t): + self.fill() + self.dispatch(t.target) + self.write(" "+self.binop[t.op.__class__.__name__]+"= ") + self.dispatch(t.value) + + def _AnnAssign(self, t): + self.fill() + if not t.simple and isinstance(t.target, ast.Name): + self.write('(') + self.dispatch(t.target) + if not t.simple and isinstance(t.target, ast.Name): + self.write(')') + self.write(": ") + self.dispatch(t.annotation) + if t.value: + self.write(" = ") + self.dispatch(t.value) + + def _Return(self, t): + self.fill("return") + if t.value: + self.write(" ") + self.dispatch(t.value) + + def _Pass(self, t): + self.fill("pass") + + def _Break(self, t): + self.fill("break") + + def _Continue(self, t): + self.fill("continue") + + def _Delete(self, t): + self.fill("del ") + interleave(lambda: self.write(", "), self.dispatch, t.targets) + + def _Assert(self, t): + self.fill("assert ") + self.dispatch(t.test) + if t.msg: + self.write(", ") + self.dispatch(t.msg) + + def _Global(self, t): + self.fill("global ") + interleave(lambda: self.write(", "), self.write, t.names) + + def _Nonlocal(self, t): + self.fill("nonlocal ") + interleave(lambda: self.write(", "), self.write, t.names) + + def _Await(self, t): + self.write("(") + self.write("await") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + + def _Yield(self, t): + self.write("(") + self.write("yield") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + + def _YieldFrom(self, t): + self.write("(") + self.write("yield from") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + + def _Raise(self, t): + self.fill("raise") + if not t.exc: + assert not t.cause + return + self.write(" ") + self.dispatch(t.exc) + if t.cause: + self.write(" from ") + self.dispatch(t.cause) + + def _Try(self, t): + self.fill("try") + self.enter() + self.dispatch(t.body) + self.leave() + for ex in t.handlers: + self.dispatch(ex) + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + if t.finalbody: + self.fill("finally") + self.enter() + self.dispatch(t.finalbody) + self.leave() + + def _ExceptHandler(self, t): + self.fill("except") + if t.type: + self.write(" ") + self.dispatch(t.type) + if t.name: + self.write(" as ") + self.write(t.name) + self.enter() + self.dispatch(t.body) + self.leave() + + def _ClassDef(self, t): + self.write("\n") + for deco in t.decorator_list: + self.fill("@") + self.dispatch(deco) + self.fill("class "+t.name) + self.write("(") + comma = False + for e in t.bases: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + for e in t.keywords: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + self.write(")") + + self.enter() + self.dispatch(t.body) + self.leave() + + def _FunctionDef(self, t): + self.__FunctionDef_helper(t, "def") + + def _AsyncFunctionDef(self, t): + self.__FunctionDef_helper(t, "async def") + + def __FunctionDef_helper(self, t, fill_suffix): + self.write("\n") + for deco in t.decorator_list: + self.fill("@") + self.dispatch(deco) + def_str = fill_suffix+" "+t.name + "(" + self.fill(def_str) + self.dispatch(t.args) + self.write(")") + if t.returns: + self.write(" -> ") + self.dispatch(t.returns) + self.enter() + self.dispatch(t.body) + self.leave() + + def _For(self, t): + self.__For_helper("for ", t) + + def _AsyncFor(self, t): + self.__For_helper("async for ", t) + + def __For_helper(self, fill, t): + self.fill(fill) + self.dispatch(t.target) + self.write(" in ") + self.dispatch(t.iter) + self.enter() + self.dispatch(t.body) + self.leave() + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _If(self, t): + self.fill("if ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + # collapse nested ifs into equivalent elifs. + while (t.orelse and len(t.orelse) == 1 and + isinstance(t.orelse[0], ast.If)): + t = t.orelse[0] + self.fill("elif ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + # final else + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _While(self, t): + self.fill("while ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _With(self, t): + self.fill("with ") + interleave(lambda: self.write(", "), self.dispatch, t.items) + self.enter() + self.dispatch(t.body) + self.leave() + + def _AsyncWith(self, t): + self.fill("async with ") + interleave(lambda: self.write(", "), self.dispatch, t.items) + self.enter() + self.dispatch(t.body) + self.leave() + + # expr + def _JoinedStr(self, t): + self.write("f") + string = io.StringIO() + self._fstring_JoinedStr(t, string.write) + self.write(repr(string.getvalue())) + + def _FormattedValue(self, t): + self.write("f") + string = io.StringIO() + self._fstring_FormattedValue(t, string.write) + self.write(repr(string.getvalue())) + + def _fstring_JoinedStr(self, t, write): + for value in t.values: + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, write) + + def _fstring_Constant(self, t, write): + assert isinstance(t.value, str) + value = t.value.replace("{", "{{").replace("}", "}}") + write(value) + + def _fstring_FormattedValue(self, t, write): + write("{") + expr = io.StringIO() + Unparser(t.value, expr) + expr = expr.getvalue().rstrip("\n") + if expr.startswith("{"): + write(" ") # Separate pair of opening brackets as "{ {" + write(expr) + if t.conversion != -1: + conversion = chr(t.conversion) + assert conversion in "sra" + write(f"!{conversion}") + if t.format_spec: + write(":") + meth = getattr(self, "_fstring_" + type(t.format_spec).__name__) + meth(t.format_spec, write) + write("}") + + def _Name(self, t): + self.write(t.id) + + def _write_constant(self, value): + if isinstance(value, (float, complex)): + # Substitute overflowing decimal literal for AST infinities. + self.write(repr(value).replace("inf", INFSTR)) + else: + self.write(repr(value)) + + def _Constant(self, t): + value = t.value + if isinstance(value, tuple): + self.write("(") + if len(value) == 1: + self._write_constant(value[0]) + self.write(",") + else: + interleave(lambda: self.write(", "), self._write_constant, value) + self.write(")") + elif value is ...: + self.write("...") + else: + if t.kind == "u": + self.write("u") + self._write_constant(t.value) + + def _List(self, t): + self.write("[") + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write("]") + + def _ListComp(self, t): + self.write("[") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write("]") + + def _GeneratorExp(self, t): + self.write("(") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write(")") + + def _SetComp(self, t): + self.write("{") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + + def _DictComp(self, t): + self.write("{") + self.dispatch(t.key) + self.write(": ") + self.dispatch(t.value) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + + def _comprehension(self, t): + if t.is_async: + self.write(" async for ") + else: + self.write(" for ") + self.dispatch(t.target) + self.write(" in ") + self.dispatch(t.iter) + for if_clause in t.ifs: + self.write(" if ") + self.dispatch(if_clause) + + def _IfExp(self, t): + self.write("(") + self.dispatch(t.body) + self.write(" if ") + self.dispatch(t.test) + self.write(" else ") + self.dispatch(t.orelse) + self.write(")") + + def _Set(self, t): + assert(t.elts) # should be at least one element + self.write("{") + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write("}") + + def _Dict(self, t): + self.write("{") + def write_key_value_pair(k, v): + self.dispatch(k) + self.write(": ") + self.dispatch(v) + + def write_item(item): + k, v = item + if k is None: + # for dictionary unpacking operator in dicts {**{'y': 2}} + # see PEP 448 for details + self.write("**") + self.dispatch(v) + else: + write_key_value_pair(k, v) + interleave(lambda: self.write(", "), write_item, zip(t.keys, t.values)) + self.write("}") + + def _Tuple(self, t): + self.write("(") + if len(t.elts) == 1: + elt = t.elts[0] + self.dispatch(elt) + self.write(",") + else: + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write(")") + + unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} + def _UnaryOp(self, t): + self.write("(") + self.write(self.unop[t.op.__class__.__name__]) + self.write(" ") + self.dispatch(t.operand) + self.write(")") + + binop = { "Add":"+", "Sub":"-", "Mult":"*", "MatMult":"@", "Div":"/", "Mod":"%", + "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", + "FloorDiv":"//", "Pow": "**"} + def _BinOp(self, t): + self.write("(") + self.dispatch(t.left) + self.write(" " + self.binop[t.op.__class__.__name__] + " ") + self.dispatch(t.right) + self.write(")") + + cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", + "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} + def _Compare(self, t): + self.write("(") + self.dispatch(t.left) + for o, e in zip(t.ops, t.comparators): + self.write(" " + self.cmpops[o.__class__.__name__] + " ") + self.dispatch(e) + self.write(")") + + boolops = {ast.And: 'and', ast.Or: 'or'} + def _BoolOp(self, t): + self.write("(") + s = " %s " % self.boolops[t.op.__class__] + interleave(lambda: self.write(s), self.dispatch, t.values) + self.write(")") + + def _Attribute(self,t): + self.dispatch(t.value) + # Special case: 3.__abs__() is a syntax error, so if t.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(t.value, ast.Constant) and isinstance(t.value.value, int): + self.write(" ") + self.write(".") + self.write(t.attr) + + def _Call(self, t): + self.dispatch(t.func) + self.write("(") + comma = False + for e in t.args: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + for e in t.keywords: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + self.write(")") + + def _Subscript(self, t): + self.dispatch(t.value) + self.write("[") + if (isinstance(t.slice, ast.Index) + and isinstance(t.slice.value, ast.Tuple) + and t.slice.value.elts): + if len(t.slice.value.elts) == 1: + elt = t.slice.value.elts[0] + self.dispatch(elt) + self.write(",") + else: + interleave(lambda: self.write(", "), self.dispatch, t.slice.value.elts) + else: + self.dispatch(t.slice) + self.write("]") + + def _Starred(self, t): + self.write("*") + self.dispatch(t.value) + + # slice + def _Ellipsis(self, t): + self.write("...") + + def _Index(self, t): + self.dispatch(t.value) + + def _Slice(self, t): + if t.lower: + self.dispatch(t.lower) + self.write(":") + if t.upper: + self.dispatch(t.upper) + if t.step: + self.write(":") + self.dispatch(t.step) + + def _ExtSlice(self, t): + if len(t.dims) == 1: + elt = t.dims[0] + self.dispatch(elt) + self.write(",") + else: + interleave(lambda: self.write(', '), self.dispatch, t.dims) + + # argument + def _arg(self, t): + self.write(t.arg) + if t.annotation: + self.write(": ") + self.dispatch(t.annotation) + + # others + def _arguments(self, t): + first = True + # normal arguments + all_args = t.posonlyargs + t.args + defaults = [None] * (len(all_args) - len(t.defaults)) + t.defaults + for index, elements in enumerate(zip(all_args, defaults), 1): + a, d = elements + if first:first = False + else: self.write(", ") + self.dispatch(a) + if d: + self.write("=") + self.dispatch(d) + if index == len(t.posonlyargs): + self.write(", /") + + # varargs, or bare '*' if no varargs but keyword-only arguments present + if t.vararg or t.kwonlyargs: + if first:first = False + else: self.write(", ") + self.write("*") + if t.vararg: + self.write(t.vararg.arg) + if t.vararg.annotation: + self.write(": ") + self.dispatch(t.vararg.annotation) + + # keyword-only arguments + if t.kwonlyargs: + for a, d in zip(t.kwonlyargs, t.kw_defaults): + if first:first = False + else: self.write(", ") + self.dispatch(a), + if d: + self.write("=") + self.dispatch(d) + + # kwargs + if t.kwarg: + if first:first = False + else: self.write(", ") + self.write("**"+t.kwarg.arg) + if t.kwarg.annotation: + self.write(": ") + self.dispatch(t.kwarg.annotation) + + def _keyword(self, t): + if t.arg is None: + self.write("**") + else: + self.write(t.arg) + self.write("=") + self.dispatch(t.value) + + def _Lambda(self, t): + self.write("(") + self.write("lambda ") + self.dispatch(t.args) + self.write(": ") + self.dispatch(t.body) + self.write(")") + + def _alias(self, t): + self.write(t.name) + if t.asname: + self.write(" as "+t.asname) + + def _withitem(self, t): + self.dispatch(t.context_expr) + if t.optional_vars: + self.write(" as ") + self.dispatch(t.optional_vars) + +def roundtrip(filename, output=sys.stdout): + with open(filename, "rb") as pyfile: + encoding = tokenize.detect_encoding(pyfile.readline)[0] + with open(filename, "r", encoding=encoding) as pyfile: + source = pyfile.read() + tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) + Unparser(tree, output) + + + +def testdir(a): + try: + names = [n for n in os.listdir(a) if n.endswith('.py')] + except OSError: + print("Directory not readable: %s" % a, file=sys.stderr) + else: + for n in names: + fullname = os.path.join(a, n) + if os.path.isfile(fullname): + output = io.StringIO() + print('Testing %s' % fullname) + try: + roundtrip(fullname, output) + except Exception as e: + print(' Failed to compile, exception is %s' % repr(e)) + elif os.path.isdir(fullname): + testdir(fullname) + +def main(args): + if args[0] == '--testdir': + for a in args[1:]: + testdir(a) + else: + for a in args: + roundtrip(a) + +if __name__=='__main__': + main(sys.argv[1:]) diff --git a/python/utils.py b/python/utils.py index 892044d..0289a5d 100644 --- a/python/utils.py +++ b/python/utils.py @@ -1,3 +1,3 @@ def concat_(*args): - """ Converts `args` into string and joines them """ - return ''.join([str(x) for x in list(args)]) + """Converts `args` into string and joines them""" + return "".join([str(x) for x in list(args)]) diff --git a/python/vimenv.py b/python/vimenv.py index 2dfb043..1923b79 100644 --- a/python/vimenv.py +++ b/python/vimenv.py @@ -3,40 +3,39 @@ class Enviroment(abc.ABC): - @property @abc.abstractmethod def plugin_root_dir(self): - """ Return absolute path to directory one level above of directory - containing python scripts. + """Return absolute path to directory one level above of directory + containing python scripts. """ @property @abc.abstractmethod def python_style(self): - """ Returns string containing docstring style: google, numpy, etc. - The style has to have corresponding template of name -