Make vimlparser.py (much more) PEP8 compliant

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+per-file-ignores =
+    # undefined names for StringReader, VimLParser, Compiler
+    py/vimlfunc.py:F821
+    py/vimlparser.py:E501

.travis.yml

@@ -23,12 +23,14 @@ jobs:
     - name: Installed Vim with checks
       env:
       - VIM_VERSION=installed
-      - MAKE_TARGET="clean_compiled check js/test py/test test/node_position/test_position.out"
+      - MAKE_TARGET="clean_compiled check checkpy js/test py/test test/node_position/test_position.out"
       - TEST_PROFILE=vim-profile-installed.txt
 
 install:
   - |
-    if [ "$VIM_VERSION" != 'installed' ]; then
+    if [ "$VIM_VERSION" = 'installed' ]; then
+      pip install --user flake8
+    else
       bash scripts/install-vim.sh
       export PATH=$HOME/vim/bin:$PATH
     fi

Makefile

@@ -16,6 +16,9 @@ check: all
 	  echo 'Compiled files were updated, but should have been included/committed.'; \
 	  exit 1; }
 
+checkpy: all
+	flake8 py
+
 test:
 	test/run.sh
 

autoload/vimlparser.vim

@@ -563,7 +563,7 @@ function! s:VimLParser.parse_command_modifiers()
       call add(modifiers, {'name': 'keepjumps'})
     elseif stridx('keeppatterns', k) == 0 && len(k) >= 5 " keepp\%[atterns]
       call add(modifiers, {'name': 'keeppatterns'})
-    elseif stridx('hide', k) == 0 && len(k) >= 3 "hid\%[e]
+    elseif stridx('hide', k) == 0 && len(k) >= 3 " hid\%[e]
       if self.ends_excmds(c)
         break
       endif
@@ -576,7 +576,7 @@ function! s:VimLParser.parse_command_modifiers()
       call add(modifiers, {'name': 'noautocmd'})
     elseif stridx('noswapfile', k) == 0 && len(k) >= 3 " :nos\%[wapfile]
       call add(modifiers, {'name': 'noswapfile'})
-    elseif stridx('rightbelow', k) == 0 && len(k) >= 6 "rightb\%[elow]
+    elseif stridx('rightbelow', k) == 0 && len(k) >= 6 " rightb\%[elow]
       call add(modifiers, {'name': 'rightbelow'})
     elseif stridx('sandbox', k) == 0 && len(k) >= 3 " san\%[dbox]
       call add(modifiers, {'name': 'sandbox'})

js/vimlparser.js

@@ -755,7 +755,7 @@ VimLParser.prototype.parse_command_modifiers = function() {
             viml_add(modifiers, {"name":"keeppatterns"});
         }
         else if (viml_stridx("hide", k) == 0 && viml_len(k) >= 3) {
-            //hid\%[e]
+            // hid\%[e]
             if (this.ends_excmds(c)) {
                 break;
             }
@@ -778,7 +778,7 @@ VimLParser.prototype.parse_command_modifiers = function() {
             viml_add(modifiers, {"name":"noswapfile"});
         }
         else if (viml_stridx("rightbelow", k) == 0 && viml_len(k) >= 6) {
-            //rightb\%[elow]
+            // rightb\%[elow]
             viml_add(modifiers, {"name":"rightbelow"});
         }
         else if (viml_stridx("sandbox", k) == 0 && viml_len(k) >= 3) {

py/pycompiler.vim

@@ -106,6 +106,7 @@ endfunction
 function s:PythonCompiler.__init__()
   let self.indent = ['']
   let self.lines = []
+  let self.in_class = 0
 endfunction
 
 function s:PythonCompiler.out(...)
@@ -324,31 +325,66 @@ function s:PythonCompiler.compile_excmd(node)
   throw 'NotImplemented: excmd'
 endfunction
 
+function s:PythonCompiler.insert_empty_lines_before_comment(count)
+  " Find start of preceding comment (block).
+  let comment_start = 0
+  let len_lines = len(self.lines)
+  if len_lines
+    while 1
+      let line = get(self.lines, comment_start - 1, '')
+      if line !~# '^\s*#'
+        break
+      endif
+      let comment_start -= 1
+      " Adjust indentation to current level.
+      let self.lines[comment_start] = substitute(line, '^\s\+', self.indent[0], '')
+    endwhile
+
+    if comment_start != 0
+      let comment_start = len_lines + comment_start
+    endif
+  endif
+
+  if comment_start
+    for c in range(a:count)
+      call insert(self.lines, '', comment_start)
+    endfor
+  else
+    for c in range(a:count)
+      call self.emptyline()
+    endfor
+  endif
+endfunction
+
 function s:PythonCompiler.compile_function(node)
   let left = self.compile(a:node.left)
   let rlist = map(a:node.rlist, 'self.compile(v:val)')
   if !empty(rlist) && rlist[-1] == '...'
     let rlist[-1] = '*a000'
   endif
+
   if left =~ '^\(VimLParser\|ExprTokenizer\|ExprParser\|LvalueParser\|StringReader\|Compiler\|RegexpParser\)\.'
     let left = matchstr(left, '\.\zs.*')
     if left == 'new'
       return
     endif
+    call self.insert_empty_lines_before_comment(1)
     call insert(rlist, 'self')
-    call self.incindent('    ')
     call self.out('def %s(%s):', left, join(rlist, ', '))
     call self.incindent('    ')
     call self.compile_body(a:node.body)
     call self.decindent()
-    call self.decindent()
   else
+    if self.in_class
+      let self.in_class = 0
+      call self.decindent()
+    endif
+    call self.insert_empty_lines_before_comment(2)
     call self.out('def %s(%s):', left, join(rlist, ', '))
     call self.incindent('    ')
     call self.compile_body(a:node.body)
     call self.decindent()
   endif
-  call self.emptyline()
 endfunction
 
 function s:PythonCompiler.compile_delfunction(node)
@@ -375,20 +411,26 @@ function s:PythonCompiler.compile_let(node)
   let right = self.compile(a:node.right)
   if a:node.left isnot s:NIL
     let left = self.compile(a:node.left)
-    if left == 'LvalueParser'
-      call self.out('class LvalueParser(ExprParser):')
-      return
-    elseif left =~ '^\(VimLParser\|ExprTokenizer\|ExprParser\|LvalueParser\|StringReader\|Compiler\|RegexpParser\)$'
-      call self.out('class %s:', left)
-      return
-    elseif left =~ '^\(VimLParser\|ExprTokenizer\|ExprParser\|LvalueParser\|StringReader\|Compiler\|RegexpParser\)\.'
+    if left ==# 'LvalueParser'
+      let class_def = 'LvalueParser(ExprParser)'
+    elseif left =~# '^\(VimLParser\|ExprTokenizer\|ExprParser\|LvalueParser\|StringReader\|Compiler\|RegexpParser\)$'
+      let class_def = left
+    elseif left =~# '^\(VimLParser\|ExprTokenizer\|ExprParser\|LvalueParser\|StringReader\|Compiler\|RegexpParser\)\.'
       let left = matchstr(left, '\.\zs.*')
-      call self.incindent('    ')
       call self.out('%s %s %s', left, op, right)
-      call self.decindent()
+      return
+    else
+      call self.out('%s %s %s', left, op, right)
       return
     endif
-    call self.out('%s %s %s', left, op, right)
+
+    if self.in_class
+      call self.decindent()
+    endif
+    call self.insert_empty_lines_before_comment(2)
+    call self.out('class %s:', class_def)
+    let self.in_class = 1
+    call self.incindent('    ')
   else
     let list = map(a:node.list, 'self.compile(v:val)')
     if a:node.rest isnot s:NIL
@@ -763,7 +805,7 @@ function s:PythonCompiler.compile_list(node)
 endfunction
 
 function s:PythonCompiler.compile_dict(node)
-  let value = map(a:node.value, 'self.compile(v:val[0]) . ":" . self.compile(v:val[1])')
+  let value = map(a:node.value, 'self.compile(v:val[0]) . ": " . self.compile(v:val[1])')
   if empty(value)
     return 'AttributeDict({})'
   else
@@ -828,14 +870,16 @@ let s:viml_builtin_functions = map(copy(s:VimLParser.builtin_functions), 'v:val.
 let s:script_dir = expand('<sfile>:h')
 function! s:convert(in, out)
   let vimlfunc = fnamemodify(s:script_dir . '/vimlfunc.py', ':p')
-  let head = readfile(vimlfunc)
+  let head = readfile(vimlfunc) + ['', '']
   try
     let r = s:StringReader.new(readfile(a:in))
     let p = s:VimLParser.new()
     let c = s:PythonCompiler.new()
     let lines = c.compile(p.parse(r))
     unlet lines[0 : index(lines, 'NIL = []') - 1]
     let tail = [
+    \   '',
+    \   '',
     \   'if __name__ == ''__main__'':',
     \   '    main()',
     \ ]

py/vimlfunc.py

@@ -3,10 +3,10 @@
 
 import sys
 import re
-import inspect
+
 
 def main():
-    use_neovim = sys.argv[1] == '--neovim'
+    use_neovim = sys.argv[1] == "--neovim"
 
     r = StringReader(viml_readfile(sys.argv[-1]))
     p = VimLParser(use_neovim)
@@ -18,139 +18,163 @@ def main():
         print(e)
         sys.exit(1)
 
+
 class VimLParserException(Exception):
     pass
 
+
 class AttributeDict(dict):
     __getattr__ = dict.__getitem__
     __setattr__ = dict.__setitem__
     __delattr__ = dict.__delitem__
 
+
 pat_vim2py = {
-  "[0-9a-zA-Z]" : "[0-9a-zA-Z]",
-  "[@*!=><&~#]" : "[@*!=><&~#]",
-  "\\<ARGOPT\\>" : "\\bARGOPT\\b",
-  "\\<BANG\\>" : "\\bBANG\\b",
-  "\\<EDITCMD\\>" : "\\bEDITCMD\\b",
-  "\\<NOTRLCOM\\>" : "\\bNOTRLCOM\\b",
-  "\\<TRLBAR\\>" : "\\bTRLBAR\\b",
-  "\\<USECTRLV\\>" : "\\bUSECTRLV\\b",
-  "\\<USERCMD\\>" : "\\bUSERCMD\\b",
-  "\\<\\(XFILE\\|FILES\\|FILE1\\)\\>" : "\\b(XFILE|FILES|FILE1)\\b",
-  "\\S" : "\\S",
-  "\\a" : "[A-Za-z]",
-  "\\d" : "\\d",
-  "\\h" : "[A-Za-z_]",
-  "\\s" : "\\s",
-  "\\v^d%[elete][lp]$" : "^d(elete|elet|ele|el|e)[lp]$",
-  "\\v^s%(c[^sr][^i][^p]|g|i[^mlg]|I|r[^e])" : "^s(c[^sr][^i][^p]|g|i[^mlg]|I|r[^e])",
-  "\\w" : "[0-9A-Za-z_]",
-  "\\w\\|[:#]" : "[0-9A-Za-z_]|[:#]",
-  "\\x" : "[0-9A-Fa-f]",
-  "^++" : "^\+\+",
-  "^++bad=\\(keep\\|drop\\|.\\)\\>" : "^\\+\\+bad=(keep|drop|.)\\b",
-  "^++bad=drop" : "^\\+\\+bad=drop",
-  "^++bad=keep" : "^\\+\\+bad=keep",
-  "^++bin\\>" : "^\\+\\+bin\\b",
-  "^++edit\\>" : "^\\+\\+edit\\b",
-  "^++enc=\\S" : "^\\+\\+enc=\\S",
-  "^++encoding=\\S" : "^\\+\\+encoding=\\S",
-  "^++ff=\\(dos\\|unix\\|mac\\)\\>" : "^\\+\\+ff=(dos|unix|mac)\\b",
-  "^++fileformat=\\(dos\\|unix\\|mac\\)\\>" : "^\\+\\+fileformat=(dos|unix|mac)\\b",
-  "^++nobin\\>" : "^\\+\\+nobin\\b",
-  "^[A-Z]" : "^[A-Z]",
-  "^\\$\\w\\+" : "^\\$[0-9A-Za-z_]+",
-  "^\\(!\\|global\\|vglobal\\)$" : "^(!|global|vglobal)$",
-  "^\\(WHILE\\|FOR\\)$" : "^(WHILE|FOR)$",
-  "^\\(vimgrep\\|vimgrepadd\\|lvimgrep\\|lvimgrepadd\\)$" : "^(vimgrep|vimgrepadd|lvimgrep|lvimgrepadd)$",
-  "^\\d" : "^\\d",
-  "^\\h" : "^[A-Za-z_]",
-  "^\\s" : "^\\s",
-  "^\\s*\\\\" : "^\\s*\\\\",
-  "^[ \\t]$" : "^[ \\t]$",
-  "^[A-Za-z]$" : "^[A-Za-z]$",
-  "^[0-9A-Za-z]$" : "^[0-9A-Za-z]$",
-  "^[0-9]$" : "^[0-9]$",
-  "^[0-9A-Fa-f]$" : "^[0-9A-Fa-f]$",
-  "^[0-9A-Za-z_]$" : "^[0-9A-Za-z_]$",
-  "^[A-Za-z_]$" : "^[A-Za-z_]$",
-  "^[0-9A-Za-z_:#]$" : "^[0-9A-Za-z_:#]$",
-  "^[A-Za-z_][0-9A-Za-z_]*$" : "^[A-Za-z_][0-9A-Za-z_]*$",
-  "^[A-Z]$" : "^[A-Z]$",
-  "^[a-z]$" : "^[a-z]$",
-  "^[vgslabwt]:$\\|^\\([vgslabwt]:\\)\\?[A-Za-z_][0-9A-Za-z_#]*$" : "^[vgslabwt]:$|^([vgslabwt]:)?[A-Za-z_][0-9A-Za-z_#]*$",
-  "^[0-7]$" : "^[0-7]$",
-  "^[0-9A-Fa-f][0-9A-Fa-f]$" : "^[0-9A-Fa-f][0-9A-Fa-f]$",
-  "^\.[0-9A-Fa-f]$" : "^\.[0-9A-Fa-f]$",
-  "^[0-9A-Fa-f][^0-9A-Fa-f]$" : "^[0-9A-Fa-f][^0-9A-Fa-f]$",
+    "[0-9a-zA-Z]": "[0-9a-zA-Z]",
+    "[@*!=><&~#]": "[@*!=><&~#]",
+    "\\<ARGOPT\\>": "\\bARGOPT\\b",
+    "\\<BANG\\>": "\\bBANG\\b",
+    "\\<EDITCMD\\>": "\\bEDITCMD\\b",
+    "\\<NOTRLCOM\\>": "\\bNOTRLCOM\\b",
+    "\\<TRLBAR\\>": "\\bTRLBAR\\b",
+    "\\<USECTRLV\\>": "\\bUSECTRLV\\b",
+    "\\<USERCMD\\>": "\\bUSERCMD\\b",
+    "\\<\\(XFILE\\|FILES\\|FILE1\\)\\>": "\\b(XFILE|FILES|FILE1)\\b",
+    "\\S": "\\S",
+    "\\a": "[A-Za-z]",
+    "\\d": "\\d",
+    "\\h": "[A-Za-z_]",
+    "\\s": "\\s",
+    "\\v^d%[elete][lp]$": "^d(elete|elet|ele|el|e)[lp]$",
+    "\\v^s%(c[^sr][^i][^p]|g|i[^mlg]|I|r[^e])":
+        "^s(c[^sr][^i][^p]|g|i[^mlg]|I|r[^e])",
+    "\\w": "[0-9A-Za-z_]",
+    "\\w\\|[:#]": "[0-9A-Za-z_]|[:#]",
+    "\\x": "[0-9A-Fa-f]",
+    "^++": r"^\+\+",
+    "^++bad=\\(keep\\|drop\\|.\\)\\>": "^\\+\\+bad=(keep|drop|.)\\b",
+    "^++bad=drop": "^\\+\\+bad=drop",
+    "^++bad=keep": "^\\+\\+bad=keep",
+    "^++bin\\>": "^\\+\\+bin\\b",
+    "^++edit\\>": "^\\+\\+edit\\b",
+    "^++enc=\\S": "^\\+\\+enc=\\S",
+    "^++encoding=\\S": "^\\+\\+encoding=\\S",
+    "^++ff=\\(dos\\|unix\\|mac\\)\\>": "^\\+\\+ff=(dos|unix|mac)\\b",
+    "^++fileformat=\\(dos\\|unix\\|mac\\)\\>":
+        "^\\+\\+fileformat=(dos|unix|mac)\\b",
+    "^++nobin\\>": "^\\+\\+nobin\\b",
+    "^[A-Z]": "^[A-Z]",
+    "^\\$\\w\\+": "^\\$[0-9A-Za-z_]+",
+    "^\\(!\\|global\\|vglobal\\)$": "^(!|global|vglobal)$",
+    "^\\(WHILE\\|FOR\\)$": "^(WHILE|FOR)$",
+    "^\\(vimgrep\\|vimgrepadd\\|lvimgrep\\|lvimgrepadd\\)$":
+        "^(vimgrep|vimgrepadd|lvimgrep|lvimgrepadd)$",
+    "^\\d": "^\\d",
+    "^\\h": "^[A-Za-z_]",
+    "^\\s": "^\\s",
+    "^\\s*\\\\": "^\\s*\\\\",
+    "^[ \\t]$": "^[ \\t]$",
+    "^[A-Za-z]$": "^[A-Za-z]$",
+    "^[0-9A-Za-z]$": "^[0-9A-Za-z]$",
+    "^[0-9]$": "^[0-9]$",
+    "^[0-9A-Fa-f]$": "^[0-9A-Fa-f]$",
+    "^[0-9A-Za-z_]$": "^[0-9A-Za-z_]$",
+    "^[A-Za-z_]$": "^[A-Za-z_]$",
+    "^[0-9A-Za-z_:#]$": "^[0-9A-Za-z_:#]$",
+    "^[A-Za-z_][0-9A-Za-z_]*$": "^[A-Za-z_][0-9A-Za-z_]*$",
+    "^[A-Z]$": "^[A-Z]$",
+    "^[a-z]$": "^[a-z]$",
+    "^[vgslabwt]:$\\|^\\([vgslabwt]:\\)\\?[A-Za-z_][0-9A-Za-z_#]*$":
+        "^[vgslabwt]:$|^([vgslabwt]:)?[A-Za-z_][0-9A-Za-z_#]*$",
+    "^[0-7]$": "^[0-7]$",
+    "^[0-9A-Fa-f][0-9A-Fa-f]$": "^[0-9A-Fa-f][0-9A-Fa-f]$",
+    r"^\.[0-9A-Fa-f]$": r"^\.[0-9A-Fa-f]$",
+    "^[0-9A-Fa-f][^0-9A-Fa-f]$": "^[0-9A-Fa-f][^0-9A-Fa-f]$",
 }
 
+
 def viml_add(lst, item):
     lst.append(item)
 
+
 def viml_call(func, *args):
     func(*args)
 
+
 def viml_char2nr(c):
     return ord(c)
 
+
 def viml_empty(obj):
     return len(obj) == 0
 
+
 def viml_equalci(a, b):
     return a.lower() == b.lower()
 
+
 def viml_eqreg(s, reg):
     return re.search(pat_vim2py[reg], s, re.IGNORECASE)
 
+
 def viml_eqregh(s, reg):
     return re.search(pat_vim2py[reg], s)
 
+
 def viml_eqregq(s, reg):
     return re.search(pat_vim2py[reg], s, re.IGNORECASE)
 
+
 def viml_escape(s, chars):
-    r = ''
+    r = ""
     for c in s:
         if c in chars:
             r += "\\" + c
         else:
             r += c
     return r
 
+
 def viml_extend(obj, item):
     obj.extend(item)
 
-def viml_insert(lst, item, idx = 0):
+
+def viml_insert(lst, item, idx=0):
     lst.insert(idx, item)
 
+
 def viml_join(lst, sep):
     return sep.join(lst)
 
+
 def viml_keys(obj):
     return obj.keys()
 
+
 def viml_len(obj):
     if type(obj) is str:
         if sys.version_info < (3, 0):
             b = bytes(obj)
         else:
-            b = bytes(obj, 'utf8')
+            b = bytes(obj, "utf8")
         return len(b)
     return len(obj)
 
+
 def viml_printf(*args):
     if len(args) == 1:
         return args[0]
     else:
         return args[0] % args[1:]
 
+
 def viml_range(start, end=None):
     if end is None:
         return range(start)
     else:
         return range(start, end + 1)
 
+
 def viml_readfile(path):
     lines = []
     f = open(path)
@@ -159,23 +183,28 @@ def viml_readfile(path):
     f.close()
     return lines
 
+
 def viml_remove(lst, idx):
     del lst[idx]
 
+
 def viml_split(s, sep):
     if sep == "\\zs":
         return s
     raise VimLParserException("NotImplemented")
 
+
 def viml_str2nr(s, base=10):
     return int(s, base)
 
+
 def viml_string(obj):
     return str(obj)
 
+
 def viml_has_key(obj, key):
     return key in obj
 
+
 def viml_stridx(a, b):
     return a.find(b)
-