Extract encoder as a package (#256)

* extract encoder

* update lockfile

* run unit tests

.github/workflows/ci.yaml

@@ -73,10 +73,50 @@ jobs:
               args: [--frozen-lockfile]
       - name: Build
         run: pnpm run build
+
+  test:
+    name: Unit Tests
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Cache node packages
+        uses: actions/cache@v2
+        env:
+          cache-name: pnpm-modules
+        with:
+          key: >-
+            ${{ runner.os }}-build-${{ env.cache-name }}-${{
+            hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ env.cache-name }}-
+            ${{ runner.os }}-build-
+            ${{ runner.os }}-
+          path: |
+            ~/.pnpm-store
+            ${{ github.workspace }}/.pnpm
+      - name: Setup Node
+        uses: actions/setup-node@v2
+        with:
+          node-version: "16"
+      - name: Setup PNPM
+        uses: pnpm/[email protected]
+        with:
+          version: 6.2.5
+          run_install: |
+            - recursive: true
+              args: [--frozen-lockfile]
+      - name: Download build artefact
+        uses: actions/download-artifact@v2
+        with:
+          name: build-artefact
+          path: .
+      - run: pnpm test
   pre-release:
     name: Pre-release
     runs-on: ubuntu-latest
-    needs: build
+    needs: test
     concurrency:
       group: pre-release
       cancel-in-progress: true
@@ -146,7 +186,7 @@ jobs:
   release:
     name: Release
     runs-on: ubuntu-latest
-    needs: build
+    needs: test
     if: startsWith(github.event.ref, 'refs/tags/v')
     concurrency:
       group: release

extension/CHANGELOG.md

@@ -2,8 +2,6 @@
 
 ## 0.18.1
 
-### Patch Changes
-
 - c735bc8: Use config from workspace configuration (correctly)
 
 ## 0.18.0

extension/package.json

@@ -383,9 +383,6 @@
           "file": "dist/server/index.node.js",
           "bundle": {
             "platform": "node",
-            "banner": {
-              "js": "console.log = console.warn = console.info = console.count = console.group = console.groupEnd = console.groupCollapsed = console.error;"
-            },
             "conditions": [
               "node",
               "import"

extension/src/GrammarlyClient.ts

@@ -82,6 +82,9 @@ export class GrammarlyClient implements Registerable {
             return 1
           },
         },
+        markdown: {
+          isTrusted: true,
+        },
       },
     )
 

fixtures/readme.md

@@ -13,4 +13,4 @@ But wait...there’s more?
 
 Grammarly Premium can give you very helpful feedback on your writing. Passive voice can be fixed by Grammarly, and it can handle classical word-choice mistakes. It can also help with inconsistencies such as switching between Email, e-mail, and email or the U.S.A. and the USA.
 
-It can even help when you wanna refine ur slang or formality level. That’s especially useful when writing for a broad audience ranging from businessmen to friends and family, don’t you think? It’ll inspect your vocabulary carefully and suggest the best word to make sure you don’t have to analyze your writing too much.
+It can even help when _you wanna_ refine ur slang or formality level. That’s especially useful when writing for a broad audience ranging from businessmen to friends and family, don’t you think? It’ll inspect your vocabulary carefully and suggest the best word to make sure you don’t have to analyze your writing too much.

jest.config.js

@@ -1,4 +1,7 @@
-module.exports = {
-  preset: 'ts-jest',
-  testEnvironment: 'node',
+// @ts-check
+/** @type {import('@jest/types').Config.InitialOptions} */
+const config = {
+  projects: ['packages/*/jest.config.js'],
 }
+
+module.exports = config

package.json

@@ -19,6 +19,7 @@
     "@rollup/plugin-node-resolve": "^13.2.1",
     "@rollup/plugin-replace": "^4.0.0",
     "@rollup/plugin-typescript": "^8.3.2",
+    "@types/jest": "^27.5.0",
     "@vscode/test-web": "^0.0.24",
     "@vuedx/monorepo-tools": "^0.2.2-next-1651055813.0",
     "esbuild": "^0.14.38",
@@ -33,7 +34,6 @@
     "tree-sitter-cli": "^0.20.6",
     "tree-sitter-html": "^0.19.0",
     "tree-sitter-markdown": "^0.7.1",
-    "ts-jest": "^27.1.4",
     "tslib": "^2.4.0",
     "typescript": "^4.6.4",
     "vsce": "^2.7.0"

packages/grammarly-languageclient/src/index.node.ts

@@ -1,4 +1,4 @@
-import { LanguageClient, ServerOptions } from 'vscode-languageclient/node'
+import { LanguageClient, ServerOptions, TransportKind } from 'vscode-languageclient/node'
 import type { GrammarlyLanguageClientOptions } from './GrammarlyLanguageClientOptions'
 import { createProtocol, Protocol } from './protocol'
 
@@ -21,9 +21,10 @@ export class GrammarlyLanguageClient extends LanguageClient {
 
 function getLanguageServerOptions(module: string): ServerOptions {
   return {
-    run: { module },
+    run: { module, transport: TransportKind.ipc },
     debug: {
       module,
+      transport: TransportKind.ipc,
       options: {
         execArgv: ['--nolazy', '--inspect=5512'],
       },

packages/grammarly-languageserver/package.json

@@ -68,6 +68,7 @@
   ],
   "dependencies": {
     "@grammarly/sdk": "^1.7.4",
+    "grammarly-richtext-encoder": "workspace:*",
     "htmlparser2": "^8.0.1",
     "idb-keyval": "^6.1.0",
     "inversify": "^6.0.1",

packages/grammarly-languageserver/src/createLanguageServer.ts

@@ -49,18 +49,6 @@ export function createLanguageServer({
     })
     const connection = getConnection()
 
-    console.log = console.debug = (...args) => {
-      connection.console.log(args.map(toString).join(' '))
-    }
-    console.error = (...args) => {
-      connection.console.error(args.map(toString).join(' '))
-    }
-    console.warn = (...args) => {
-      connection.console.warn(args.map(toString).join(' '))
-    }
-    console.info = (...args) => {
-      connection.console.info(args.map(toString).join(' '))
-    }
     container.bind(CONNECTION).toConstantValue(connection)
     container.bind(SERVER).toConstantValue(capabilities)
 

packages/grammarly-languageserver/src/services/CodeActionService.ts

@@ -93,6 +93,7 @@ export class CodeActionService implements Registerable {
         suggestionId,
         replacementId,
       })
+      console.log(JSON.stringify(edit, null, 2))
       const range = document.findOriginalRange(edit.range.start, edit.range.end)
       const newText = document.toText(edit.content)
 

packages/grammarly-languageserver/src/services/DiagnosticsService.ts

@@ -1,4 +1,4 @@
-import { Markup, MarkupChild, Suggestion, SuggestionId } from '@grammarly/sdk'
+import { Suggestion, SuggestionId } from '@grammarly/sdk'
 import { inject, injectable } from 'inversify'
 import type { Connection, Diagnostic, DiagnosticSeverity, Disposable, Range } from 'vscode-languageserver'
 import { CONNECTION } from '../constants'
@@ -123,47 +123,3 @@ export class DiagnosticsService implements Registerable {
     }
   }
 }
-
-function toText(markup: Markup): string {
-  function stringify(node: MarkupChild): string {
-    if (typeof node === 'string') return node
-
-    switch (node.type) {
-      case 'ul':
-        return `\n${node.children.map((node) => `- ${stringify(node)}`).join('\n')}\n`
-      default:
-        return node.children.map(stringify).join('')
-    }
-  }
-
-  return markup.map(stringify).join('')
-}
-
-export function toMarkdown(markup: Markup): string {
-  let indent = 0
-  function stringify(node: MarkupChild): string {
-    if (typeof node === 'string') return node
-
-    switch (node.type) {
-      case 'ul':
-        try {
-          indent += 2
-          return `\n${node.children.map(stringify).join('\n')}`
-        } finally {
-          indent -= 2
-        }
-      case 'li':
-        return ' '.repeat(indent - 2) + `- ${node.children.map(stringify).join('')}`
-      case 'del':
-        return ` ~~${node.children.map(stringify).join('')}~~ `
-      case 'em':
-        return `_${node.children.map(stringify).join('')}_`
-      case 'strong':
-        return `**${node.children.map(stringify).join('')}**`
-      default:
-        return node.children.map(stringify).join('')
-    }
-  }
-
-  return markup.map(stringify).join('')
-}

packages/grammarly-languageserver/src/services/DocumentService.ts

@@ -13,9 +13,7 @@ import Parser from 'web-tree-sitter'
 import { CLIENT_INITIALIZATION_OPTIONS, CONNECTION, GRAMMARLY_SDK, SERVER, TEXT_DOCUMENTS_FACTORY } from '../constants'
 import { InitializationOptions } from '../interfaces/InitializationOptions'
 import { Registerable } from '../interfaces/Registerable'
-import { SourceMap } from '../interfaces/SourceMap'
-import { Transformer } from '../interfaces/Transformer'
-import { createParser, transformers } from '../languages'
+import { createParser, transformers, SourceMap, Transformer } from 'grammarly-richtext-encoder'
 import { ConfigurationService } from './ConfigurationService'
 
 @injectable()
@@ -183,12 +181,10 @@ export class GrammarlyDocument {
   public findOriginalOffset(offset: number): number {
     if (this.#context == null) return offset
     const map = this.#context.sourcemap
-    const index = binarySearchLowerBound(0, map.length - 1, (index) => map[index][1] <= offset)
+    const index = binarySearchLowerBound(0, map.length - 1, (index) => map[index][1] < offset)
     const node = map[index]
     if (node == null) return 0
-    const original = node[0] + (offset - node[1])
-
-    return original
+    return node[0] + Math.max(0, offset - node[1])
   }
 
   public findOriginalRange(start: number, end: number): Range {

packages/grammarly-languageserver/src/services/HoverService.ts

@@ -2,8 +2,9 @@ import { inject, injectable } from 'inversify'
 import type { Connection, Disposable, ServerCapabilities } from 'vscode-languageserver'
 import { CONNECTION, SERVER } from '../constants'
 import { Registerable } from '../interfaces/Registerable'
-import { DiagnosticsService, toMarkdown } from './DiagnosticsService'
+import { DiagnosticsService } from './DiagnosticsService'
 import { DocumentService } from './DocumentService'
+import { toMarkdown } from './toMarkdown'
 
 @injectable()
 export class HoverService implements Registerable {
@@ -35,19 +36,21 @@ export class HoverService implements Registerable {
       const diagnostic = diagnostics[0]
       if (diagnostic == null) return null
 
+      const contents = `**${diagnostic.suggestion.title.trim()}**\n\n${toMarkdown(
+        diagnostic.suggestion.description,
+      ).trim()}\n\n\n${
+        diagnostic.suggestion.replacements.length === 1
+          ? `… ${toMarkdown(diagnostic.suggestion.replacements[0].preview).trim()} …`
+          : diagnostic.suggestion.replacements
+              .map((replacement) => `1. … ${toMarkdown(replacement.preview).trim()} …\n`)
+              .join('')
+      }`
+
       return {
         range: diagnostic.diagnostic.range,
         contents: {
           kind: 'markdown',
-          value: `**${diagnostic.suggestion.title.trim()}**\n\n${toMarkdown(
-            diagnostic.suggestion.description,
-          ).trim()}\n\n\n${
-            diagnostic.suggestion.replacements.length === 1
-              ? `…${toMarkdown(diagnostic.suggestion.replacements[0].preview).trim()}…`
-              : diagnostic.suggestion.replacements
-                  .map((replacement) => `1. …${toMarkdown(replacement.preview).trim()}…\n`)
-                  .join('')
-          }`,
+          value: contents,
         },
       }
     })