merge: add new bib parser in lua

This also has an improvement to the existing Vimscript implementation.

refer: #2816, #2786

.luarc.json

@@ -0,0 +1,13 @@
+{
+  "$schema": "https://raw.githubusercontent.com/LuaLS/vscode-lua/master/setting/schema.json",
+  "runtime": {
+    "version": "LuaJIT"
+  },
+  "workspace": {
+    "library": [
+      "$VIMRUNTIME",
+      "${3rd}/luv/library"
+    ],
+    "checkThirdParty": false
+  }
+}

.stylua.toml

@@ -0,0 +1,5 @@
+column_width = 80
+indent_type = "Spaces"
+indent_width = 2
+quote_style = "AutoPreferDouble"
+call_parentheses = "None"

autoload/vimtex/context/cite.vim

@@ -48,7 +48,10 @@ function! s:handler.get_actions() abort dict " {{{1
   call vimtex#paths#pushd(b:vimtex.root)
   let l:entries = []
   for l:file in vimtex#bib#files()
-    let l:entries += vimtex#parser#bib(l:file, {'backend': 'vim'})
+    let l:entries += vimtex#parser#bib(
+          \ l:file,
+          \ {'backend': has('nvim') ? 'lua' : 'vim'}
+          \)
   endfor
   call vimtex#paths#popd()
 
@@ -119,7 +122,7 @@ function! s:actions.show() abort dict " {{{1
         \ ['Normal', ','],
         \])
 
-  for l:x in ['key', 'type', 'vimtex_lnum', 'vimtex_file']
+  for l:x in ['key', 'type', 'source_lnum', 'source_file']
     if has_key(l:entry, l:x)
       call remove(l:entry, l:x)
     endif
@@ -144,10 +147,10 @@ endfunction
 
 " }}}1
 function! s:actions.edit() abort dict " {{{1
-  execute 'edit' self.entry.vimtex_file
+  execute 'edit' self.entry.source_file
   filetype detect
 
-  call vimtex#pos#set_cursor(self.entry.vimtex_lnum, 0)
+  call vimtex#pos#set_cursor(self.entry.source_lnum, 0)
   normal! zv
 endfunction
 

autoload/vimtex/options.vim

@@ -311,7 +311,9 @@ function! vimtex#options#init() abort " {{{1
   call s:init_option('vimtex_lint_chktex_ignore_warnings',
         \ '-n1 -n3 -n8 -n25 -n36')
 
-  call s:init_option('vimtex_parser_bib_backend', 'bibtex')
+  call s:init_option('vimtex_parser_bib_backend',
+        \ has('nvim') ? 'lua' : 'bibtex'
+        \)
   call s:init_option('vimtex_parser_cmd_separator_check',
         \ 'vimtex#cmd#parser_separator_check')
 

autoload/vimtex/parser/bib.vim

@@ -250,6 +250,18 @@ endfunction
 
 " }}}1
 
+function! s:parse_with_lua(file) abort " {{{1
+  if !has('nvim')
+    call vimtex#log#error(
+          \ 'bib parser backend "lua" only works with neovim!')
+    return []
+  endif
+
+  return luaeval('require("vimtex.bibparser").parse(_A)', a:file)
+endfunction
+
+" }}}1
+
 function! s:parse_with_vim(file) abort " {{{1
   " Adheres to the format description found here:
   " http://www.bibtex.org/Format/
@@ -258,117 +270,111 @@ function! s:parse_with_vim(file) abort " {{{1
     return []
   endif
 
-  let l:current = {}
+  let l:items = []
   let l:strings = {}
-  let l:entries = []
+
+  let l:item = {}
   let l:lnum = 0
   for l:line in readfile(a:file)
     let l:lnum += 1
 
-    if empty(l:current)
-      if s:parse_type(a:file, l:lnum, l:line, l:current, l:strings, l:entries)
-        let l:current = {}
-      endif
-      continue
-    endif
+    let l:item = empty(l:item)
+          \ ? s:parse_head(a:file, l:lnum, l:line)
+          \ : s:parse_tail(l:item, l:line)
 
-    if l:current.type ==# 'string'
-      if s:parse_string(l:line, l:current, l:strings)
-        let l:current = {}
-      endif
-    else
-      if s:parse_entry(l:line, l:current, l:entries)
-        let l:current = {}
+    if has_key(l:item, 'parsed')
+      if l:item.type == "string"
+        let [l:key, l:value] = s:parse_string(l:item.body)
+        if !empty(l:key)
+          let l:strings[l:key] = l:value
+        endif
+      else
+        call add(l:items, l:item)
       endif
+      let l:item = {}
     endif
   endfor
 
-  return map(l:entries, 's:parse_entry_body(v:val, l:strings)')
+  return map(l:items, 's:parse_item(v:val, l:strings)')
 endfunction
 
 " }}}1
 
-function! s:parse_type(file, lnum, line, current, strings, entries) abort " {{{1
+function! s:parse_head(file, lnum, line) abort " {{{1
   let l:matches = matchlist(a:line, '\v^\@(\w+)\s*\{\s*(.*)')
-  if empty(l:matches) | return 0 | endif
+  if empty(l:matches) | return {} | endif
 
   let l:type = tolower(l:matches[1])
-  if index(['preamble', 'comment'], l:type) >= 0 | return 0 | endif
-
-  let a:current.level = 1
-  let a:current.body = ''
-  let a:current.vimtex_file = a:file
-  let a:current.vimtex_lnum = a:lnum
+  if l:type == 'preamble' || l:type == 'comment' | return {} | endif
+
+  return s:parse_tail({
+        \ 'level': 1,
+        \ 'body': '',
+        \ 'source_file': a:file,
+        \ 'source_lnum': a:lnum,
+        \ 'type': l:type,
+        \}, l:matches[2])
+endfunction
 
-  if l:type ==# 'string'
-    return s:parse_string(l:matches[2], a:current, a:strings)
+" }}}1
+function! s:parse_tail(item, line) abort " {{{1
+  let a:item.level += s:count(a:line, '{') - s:count(a:line, '}')
+  if a:item.level > 0
+    let a:item.body .= a:line
   else
-    let l:matches = matchlist(l:matches[2], '\v^([^, ]*)\s*,\s*(.*)')
-    let a:current.type = l:type
-    let a:current.key = l:matches[1]
-
-    return empty(l:matches[2])
-          \ ? 0
-          \ : s:parse_entry(l:matches[2], a:current, a:entries)
+    let a:item.body .= matchstr(a:line, '.*\ze}')
+    let a:item.parsed = v:true
   endif
+
+  return a:item
 endfunction
 
 " }}}1
-function! s:parse_string(line, string, strings) abort " {{{1
-  let a:string.level += s:count(a:line, '{') - s:count(a:line, '}')
-  if a:string.level > 0
-    let a:string.body .= a:line
-    return 0
+function! s:parse_string(raw_string) abort " {{{1
+  let l:matches = matchlist(a:raw_string, '\v^\s*(\S+)\s*\=\s*"(.*)"\s*$')
+  if !empty(l:matches) && !empty(l:matches[1])
+    return [l:matches[1], l:matches[2]]
   endif
 
-  let a:string.body .= matchstr(a:line, '.*\ze}')
-
-  let l:matches = matchlist(a:string.body, '\v^\s*(\w+)\s*\=\s*"(.*)"\s*$')
+  let l:matches = matchlist(a:raw_string, '\v^\s*(\S+)\s*\=\s*\{(.*)\}\s*$')
   if !empty(l:matches) && !empty(l:matches[1])
-    let a:strings[l:matches[1]] = l:matches[2]
+    return [l:matches[1], l:matches[2]]
   endif
 
-  return 1
+  return ['', '']
 endfunction
 
 " }}}1
-function! s:parse_entry(line, entry, entries) abort " {{{1
-  let a:entry.level += s:count(a:line, '{') - s:count(a:line, '}')
-  if a:entry.level > 0
-    let a:entry.body .= a:line
-    return 0
-  endif
 
-  let a:entry.body .= matchstr(a:line, '.*\ze}')
+function! s:parse_item(item, strings) abort " {{{1
+  let l:parts = matchlist(a:item.body, '\v^([^, ]*)\s*,\s*(.*)')
 
-  call add(a:entries, a:entry)
-  return 1
-endfunction
-
-" }}}1
+  let a:item.key = l:parts[1]
+  if empty(a:item.key) | return {} | endif
 
-function! s:parse_entry_body(entry, strings) abort " {{{1
-  unlet a:entry.level
+  unlet a:item.level
+  unlet a:item.body
+  unlet a:item.parsed
 
-  let l:key = ''
-  let l:pos = matchend(a:entry.body, '^\s*')
-  while l:pos >= 0
-    if empty(l:key)
-      let [l:key, l:pos] = s:get_key(a:entry.body, l:pos)
+  let l:body = l:parts[2]
+  let l:tag = ''
+  let l:head = 0
+  while l:head >= 0
+    if empty(l:tag)
+      let [l:tag, l:head] = s:get_tag_name(l:body, l:head)
     else
-      let [l:value, l:pos] = s:get_value(a:entry.body, l:pos, a:strings)
-      let a:entry[l:key] = l:value
-      let l:key = ''
+      let [l:value, l:head] = s:get_tag_value(l:body, l:head, a:strings)
+      let a:item[l:tag] = l:value
+      let l:tag = ''
     endif
   endwhile
 
-  unlet a:entry.body
-  return a:entry
+  return a:item
 endfunction
 
 " }}}1
-function! s:get_key(body, head) abort " {{{1
-  " Parse the key part of a bib entry tag.
+function! s:get_tag_name(body, head) abort " {{{1
+  " Parse the name part of a bib entry tag.
   " Assumption: a:body is left trimmed and either empty or starts with a key.
   " Returns: The key and the remaining part of the entry body.
 
@@ -379,7 +385,7 @@ function! s:get_key(body, head) abort " {{{1
 endfunction
 
 " }}}1
-function! s:get_value(body, head, strings) abort " {{{1
+function! s:get_tag_value(body, head, strings) abort " {{{1
   " Parse the value part of a bib entry tag, until separating comma or end.
   " Assumption: a:body is left trimmed and either empty or starts with a value.
   " Returns: The value and the remaining part of the entry body.
@@ -393,15 +399,16 @@ function! s:get_value(body, head, strings) abort " {{{1
     let l:value = matchstr(a:body, '^\d\+', a:head)
     let l:head = matchend(a:body, '^\s*,\s*', a:head + len(l:value))
     return [l:value, l:head]
-  else
-    return s:get_value_string(a:body, a:head, a:strings)
   endif
 
-  return ['s:get_value failed', -1]
+  return s:get_tag_value_concat(a:body, a:head, a:strings, "")
 endfunction
 
 " }}}1
-function! s:get_value_string(body, head, strings) abort " {{{1
+function! s:get_tag_value_concat(body, head, strings, pre_value) abort " {{{1
+  let l:value = ""
+  let l:head = a:head
+
   if a:body[a:head] ==# '{'
     let l:sum = 1
     let l:i1 = a:head + 1
@@ -420,27 +427,24 @@ function! s:get_value_string(body, head, strings) abort " {{{1
   elseif a:body[a:head] ==# '"'
     let l:index = match(a:body, '\\\@<!"', a:head+1)
     if l:index < 0
-      return ['s:get_value_string failed', '']
+      return ['s:get_tag_value_concat failed', -1]
     endif
 
     let l:value = a:body[a:head+1:l:index-1]
     let l:head = matchend(a:body, '^\s*', l:index+1)
-    return [l:value, l:head]
   elseif a:body[a:head:] =~# '^\w'
-    let l:value = matchstr(a:body, '^\w\+', a:head)
+    let l:value = matchstr(a:body, '^\w[0-9a-zA-Z_-]*', a:head)
     let l:head = matchend(a:body, '^\s*', a:head + strlen(l:value))
     let l:value = get(a:strings, l:value, '@(' . l:value . ')')
-  else
-    let l:head = a:head
   endif
 
   if a:body[l:head] ==# '#'
     let l:head = matchend(a:body, '^\s*', l:head + 1)
-    let [l:vadd, l:head] = s:get_value_string(a:body, l:head, a:strings)
-    let l:value .= l:vadd
+    return s:get_tag_value_concat(
+          \ a:body, l:head, a:strings, a:pre_value . l:value)
   endif
 
-  return [l:value, matchend(a:body, '^,\s*', l:head)]
+  return [a:pre_value . l:value, matchend(a:body, '^,\s*', l:head)]
 endfunction
 
 " }}}1

doc/vimtex.txt

@@ -1310,8 +1310,14 @@ OPTIONS                                                        *vimtex-options*
   This option sets the desired default backend for parsing bibliographies.
   This is used e.g. for gathering completion candidates. Possible values:
 
-    `bibtex`:   The fastest, but most hacky solution. Should work well in most
-              cases.
+    `bibtex`:   The fastest, but most "hacky" solution. Still, time has proved
+              that this works well!
+
+    `vim`:      The slowest but perhaps most robust solution, as it does not
+              require any external utilities.
+
+    `lua`:      A Lua implementation of the Vim backend. About as fast as the
+              `bibtex` parser, but this only works on Neovim.
 
     `bibparse`: Also fast, but might be more robust.
 
@@ -1335,17 +1341,17 @@ OPTIONS                                                        *vimtex-options*
                     (see |if_pyth| and |py3|) and that the `bibtexparser`
                     Python module is installed and available.
 
-    `vim`:      The slowest but perhaps most robust solution, as it does not
-              require any external utilities.
-
   Some people may want to conditionally change this option if a backend is
   available. For example: >vim
 
     if executable('bibparse')
       let g:vimtex_parser_bib_backend = 'bibparse'
     endif
 <
-  Default value: `bibtex`
+  Default value:
+
+    Vim:    `bibtex`
+    Neovim: `lua`
 
 *g:vimtex_parser_cmd_separator_check*
   This option specifies the policy for deciding whether successive groups of