Generate operators for all editors (#1950)

Currently, operator names are generated only for vscode only using the command `cabal run swarm:swarm-docs -- editors --code`.
With this PR, I intend to bring that behaviour to all the editors.

Changes include:
- `cabal run swarm:swarm-docs -- editors` command now supports `vim` as well.
- `operatorNames` can generate operator list catering to all the editors supporting Swarm.
- Update operator list in `swarm-mode.el`, `swarm.vim` and `swarm.tmLanguage.yaml`.

How to test emacs syntax:
- Open `editors/emacs/swarm-mode.el` in emacs.
- Then `M-x eval-buffer`
- Open up any of the `.sw` file under `examples`.
- Then `M-x swarm-mode`

How to test vim syntax:
- Copy swarm.vim to vim directory using `cp editors/vim/swarm.vim ~/.vim/syntax/sw.vim`
- Setup auto detect in vim. `echo 'autocmd BufRead,BufNewFile *.sw set filetype=sw' > ~/.vim/ftdetect/sw.vim`
- Open up any of the `.sw` files under `examples`. (Also ensure that you have syntax on in vim. `ESC :syntax on`)

app/doc/Main.hs

@@ -28,6 +28,7 @@ cliParser =
       [ pure Nothing
       , Just VSCode <$ switch (long "code" <> help "Generate for the VS Code editor")
       , Just Emacs <$ switch (long "emacs" <> help "Generate for the Emacs editor")
+      , Just Vim <$ switch (long "vim" <> help "Generate for the Vim editor")
       ]
   address :: Parser PageAddress
   address =

editors/emacs/swarm-mode.el

@@ -25,7 +25,27 @@
   "Syntax table for `swarm-mode'.")
 
 (defvar swarm-mode-operators-regexp
-  (regexp-opt '(":" "->" "=" "<-" "+" "*" "/" "-") t)
+  (regexp-opt
+   '(
+     "-"
+     "=="
+     "!="
+     "<"
+     ">"
+     "<="
+     ">="
+     "||"
+     "&&"
+     "+"
+     "-"
+     "*"
+     "/"
+     "^"
+     "++"
+     "$"
+     ":"
+     )
+   t)
   "Regexp that recognizes operators for swarm language.")
 
 (defvar swarm-mode-commands-regexp

editors/vim/swarm.vim

@@ -2,13 +2,13 @@ syn keyword Keyword def tydef rec end let in require
 syn keyword Builtins self parent base if inl inr case fst snd force undefined fail not format chars split charat tochar key
 syn keyword Command noop wait selfdestruct move backup volume path push stride turn grab harvest sow ignite place ping give equip unequip make has equipped count drill use build salvage reprogram say listen log view appear create halt time scout whereami waypoint structure floorplan hastag tagmembers detect resonate density sniff chirp watch surveil heading blocked scan upload ishere isempty meet meetall whoami setname random run return try swap atomic instant installkeyhandler teleport as robotnamed robotnumbered knows
 syn keyword Direction east north west south down forward left back right
-syn keyword Type "\<[A-Z][a-zA-Z_]*\>"
+syn match Type "\<[A-Z][a-zA-Z_]*\>"
+syn match Operators "[-=!<>|&+*/^$:]"
 
 
 syn match Comment "//.*$"
 syn region MultilineComment start="/\*" end="\*/"
 syn match Brackets "[\[\]\(\)\{\}]"
-syn match Colon ":"
 syn match String "\".*\""
 syn match Number "\<[-]\=\d\+\>"
 
@@ -19,6 +19,6 @@ hi def link Direction Function
 hi def link Comment Comment
 hi def link MultilineComment Comment
 hi def link Brackets Keyword
-hi def link Colon Keyword
-hi def link String String 
+hi def link Operators Keyword
+hi def link String String
 hi def link Number Number

editors/vscode/syntaxes/swarm.tmLanguage.yaml

@@ -66,7 +66,7 @@ repository:
       # ---------------------------------------------
       operator:
         name: keyword.operator
-        match: '-|==|!=|<|>|<=|>=|\|\||&&|\+|-|\*|/(?![/|*])|\^|\+\+|\$'
+        match: '-|==|!=|<|>|<=|>=|\|\||&&|\+|-|\*|/(?![/|*])|\^|\+\+|\$|:'
       in:
         name: keyword.control.dictionary.let.in
         match: \b(in)\b

example/list.sw

@@ -26,7 +26,7 @@
 //
 // It is the callers responsibility to make sure a program using this
 // "type" is type safe. Notably 2 == [0] != [] == 0 but [] !! x == 0.
-// 
+//
 // TODO: once #153 is resolved, add types to definitions
 //
 // type ListI = Int
@@ -39,16 +39,16 @@
 // chunks each prefixed by 1bit that marks if the byte is last in
 // the header (0=YES).
 
-/* EXAMPLE - [short_x,long_y] - concretly e.g. [42, 2^(2^7)] 
+/* EXAMPLE - [short_x,long_y] - concretly e.g. [42, 2^(2^7)]
 
 0   < len short_x < 2^7
-2^7 < len long_y  < 2^14 
+2^7 < len long_y  < 2^14
 
 cons short_x   $          cons long_y          $ nil
 vvvvvvvvvvvv       vvvvvvvvvvvvvvvvvvvvvvvvv     vvv
   0|len x|x    |   1|len y%2^7|0|len y/2^7|y   |  0
 ^^^^^^^^^^^^       ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  head                         tail            
+  head                         tail
 */
 
 /*******************************************************************/
@@ -123,7 +123,7 @@ end
 /*                         LIST FUNCTIONS                          */
 /*******************************************************************/
 
-// headTail : ListI -> {Int} * {ListI} 
+// headTail : ListI -> {Int} * {ListI}
 def headTail = \xs.
   let sign = mod xs 2 in
   let ns = xs / 2 in
@@ -140,7 +140,7 @@ def head : Int -> Int = \xs.
   force $ fst $ headTail xs
 end
 
-// tail : ListI -> ListI 
+// tail : ListI -> ListI
 def tail = \xs.
   force $ snd $ headTail xs
 end
@@ -226,19 +226,19 @@ def testLIST =
   assert (ln1 == 517)                "[-1] ~ 517";
   assert (head ln1 == -1)            "head [-1] == -1";
   assert (tail ln1 == nil)           "tail [-1] == []";
-  
+
   log "check [42]";
   let l42 = cons 42 nil in
   assert (l42 == 21528)              "[42] ~ 21528";
   assert (head l42 == 42)            "head [42] == 42";
   assert (tail l42 == nil)           "tail [42] == []";
-  
+
   log "check [499672]";
   let l499672 = cons 499672 nil in
   assert (l499672 == 255832140)      "[499672] ~ 255832140";
   assert (head l499672 == 499672)    "head [499672] == 499672";
   assert (tail l499672 == nil)       "tail [499672] == []";
-  
+
   log "check [1,0]";
   let l1_0 = cons 1 l0 in
   assert (l1_0 == 4612)              "[1,0] ~ 4612";
@@ -304,7 +304,7 @@ def testLIST_BIG =
   let lbiggest = cons bigger lbig in
   assert (head lbiggest == bigger)  "head [bigger,big] == bigger";
   assert (tail lbiggest == lbig)    "tail [bigger,big] == [big]";
-  
+
   log "OK - ALL TEST PASSED";
 end
 

src/swarm-doc/Swarm/Doc/Gen.hs

@@ -97,26 +97,29 @@ generateDocs = \case
 generateEditorKeywords :: EditorType -> IO ()
 generateEditorKeywords = \case
   Emacs -> do
-    putStrLn "(x-builtins '("
-    T.putStr $ builtinFunctionList Emacs
-    putStrLn "))\n(x-commands '("
+    putStrLn "(defvar swarm-mode-builtins '("
+    T.putStr $ builtinFunctionList Emacs <> "))"
+    putStrLn "\n(defvar swarm-mode-commands '("
     T.putStr $ keywordsCommands Emacs
-    T.putStr $ keywordsDirections Emacs
-    putStrLn "))"
+    T.putStr $ keywordsDirections Emacs <> "))"
+    putStrLn "\n (defvar swarm-mode-operators '("
+    T.putStr $ operatorNames Emacs <> "))"
   VSCode -> do
     putStrLn "Functions and commands:"
     T.putStrLn $ builtinFunctionList VSCode <> "|" <> keywordsCommands VSCode
     putStrLn "\nDirections:"
     T.putStrLn $ keywordsDirections VSCode
     putStrLn "\nOperators:"
-    T.putStrLn operatorNames
+    T.putStrLn $ operatorNames VSCode
   Vim -> do
-    putStr "syn keyword Builtins "
+    putStrLn "syn keyword Builtins "
     T.putStr $ builtinFunctionList Vim
-    putStr "\nsyn keyword Command "
+    putStrLn "\nsyn keyword Command "
     T.putStr $ keywordsCommands Vim
-    putStr "\nsyn keyword Direction "
-    T.putStrLn $ keywordsDirections Vim
+    putStrLn "\nsyn keyword Direction "
+    T.putStr $ keywordsDirections Vim
+    putStrLn "\nsyn match Operators "
+    T.putStr $ "[" <> operatorNames Vim <> "]"
 
 -- ----------------------------------------------------------------------------
 -- GENERATE SPECIAL KEY NAMES

src/swarm-doc/Swarm/Doc/Keyword.hs

@@ -16,14 +16,15 @@ module Swarm.Doc.Keyword (
   builtinFunctionList,
 ) where
 
+import Data.List (nub)
 import Data.Text (Text)
 import Data.Text qualified as T
 import Swarm.Doc.Util
 import Swarm.Language.Syntax.Direction
 import Swarm.Util (quote)
 
 -- | An enumeration of the editors supported by Swarm (currently,
---   Emacs and VS Code).
+--   Emacs, VS Code and Vim).
 data EditorType = Emacs | VSCode | Vim
   deriving (Eq, Show, Enum, Bounded)
 
@@ -45,12 +46,26 @@ keywordsDirections :: EditorType -> Text
 keywordsDirections e = editorList e $ map directionSyntax allDirs
 
 -- | A list of the names of all the operators in the language.
-operatorNames :: Text
-operatorNames = T.intercalate "|" $ map (escape . constSyntax) operators
+-- These are reflective of how the different editors treat operators,
+-- keywords, symbols etc differently.
+-- In order to get the list of operators supported by Swarm language
+-- irrespective of an editor, @map constSyntax operators@ should suffice.
+operatorNames :: EditorType -> Text
+operatorNames e = case e of
+  Emacs -> editorList e $ map constSyntax operators <> extraOperators
+  -- Vim needs a list of unique characters that can be matched over using a regex
+  Vim -> T.pack . nub . T.unpack . T.concat $ map constSyntax operators <> extraOperators
+  VSCode -> editorList e $ map (escape . constSyntax) operators <> extraOperators
  where
-  special :: String
-  special = "*+$[]|^"
   slashNotComment = \case
     '/' -> "/(?![/|*])"
     c -> T.singleton c
+
+  special :: String
+  special = "*+$[]|^"
+
+  -- Extra operators appearing in different places. Eg: Type signatures.
+  extraOperators :: [Text]
+  extraOperators = [":"]
+
   escape = T.concatMap (\c -> if c `elem` special then T.snoc "\\" c else slashNotComment c)

test/integration/Main.hs

@@ -531,28 +531,30 @@ testEditorFiles =
     "editors"
     [ testGroup
         "VS Code"
-        [ testTextInVSCode "operators" (const Keyword.operatorNames)
+        [ testTextInVSCode "operators" Keyword.operatorNames
         , testTextInVSCode "builtin" Keyword.builtinFunctionList
         , testTextInVSCode "commands" Keyword.keywordsCommands
         , testTextInVSCode "directions" Keyword.keywordsDirections
         ]
     , testGroup
         "Emacs"
-        [ testTextInEmacs "builtin" Keyword.builtinFunctionList
+        [ testTextInEmacs "operators" Keyword.operatorNames
+        , testTextInEmacs "builtin" Keyword.builtinFunctionList
         , testTextInEmacs "commands" Keyword.keywordsCommands
         , testTextInEmacs "directions" Keyword.keywordsDirections
         ]
     , testGroup
         "Vim"
-        [ testTextInVim "builtin" Keyword.builtinFunctionList
+        [ testTextInVim "operators" Keyword.operatorNames
+        , testTextInVim "builtin" Keyword.builtinFunctionList
         , testTextInVim "commands" Keyword.keywordsCommands
         , testTextInVim "directions" Keyword.keywordsDirections
         ]
     ]
  where
   testTextInVSCode name tf = testTextInFile False name (tf VSCode) "editors/vscode/syntaxes/swarm.tmLanguage.yaml"
   testTextInEmacs name tf = testTextInFile True name (tf Emacs) "editors/emacs/swarm-mode.el"
-  testTextInVim name tf = testTextInFile True name (tf Vim) "editors/vim/swarm.vim"
+  testTextInVim name tf = testTextInFile False name (tf Vim) "editors/vim/swarm.vim"
   testTextInFile :: Bool -> String -> Text -> FilePath -> TestTree
   testTextInFile whitespace name t fp = testCase name $ do
     let removeLW' = T.unlines . map (T.dropWhile isSpace) . T.lines