feat(bundle-utils): support AST generation for JIT compilation

intlify · Jun 29, 2023 · 42dcbcb · 42dcbcb
1 parent 42260f0
commit 42dcbcb
Show file tree

Hide file tree

Showing 16 changed files with 1,238 additions and 290 deletions.
diff --git a/packages/bundle-utils/package.json b/packages/bundle-utils/package.json
@@ -18,8 +18,8 @@
     }
   },
   "dependencies": {
-    "@intlify/message-compiler": "9.3.0-beta.17",
-    "@intlify/shared": "9.3.0-beta.17",
+    "@intlify/message-compiler": "9.3.0-beta.20",
+    "@intlify/shared": "9.3.0-beta.20",
     "acorn": "^8.8.2",
     "escodegen": "^2.0.0",
     "estree-walker": "^2.0.2",

diff --git a/packages/bundle-utils/src/codegen.ts b/packages/bundle-utils/src/codegen.ts
@@ -5,7 +5,12 @@ import {
   MappedPosition,
   MappingItem
 } from 'source-map'
-import { format, escapeHtml as sanitizeHtml, isBoolean } from '@intlify/shared'
+import {
+  format,
+  escapeHtml as sanitizeHtml,
+  isBoolean,
+  friendlyJSONstringify
+} from '@intlify/shared'
 
 import type { RawSourceMap } from 'source-map'
 import type {
@@ -63,6 +68,7 @@ export interface CodeGenOptions {
   allowDynamic?: boolean
   strictMessage?: boolean
   escapeHtml?: boolean
+  jit?: boolean
   onWarn?: (msg: string) => void
   onError?: (
     msg: string,
@@ -106,16 +112,19 @@ export interface CodeGenerator {
   ): void
 }
 
-/**
- * @internal
- */
 export interface CodeGenResult<ASTNode, CodeGenError extends Error = Error> {
   code: string
   ast: ASTNode
   errors?: CodeGenError[]
   map?: RawSourceMap
 }
 
+export type CodeGenFunction = (
+  msg: string,
+  options: CodeGenOptions,
+  path?: string[]
+) => CodeGenResult<ResourceNode>
+
 export function createCodeGenerator(
   options: CodeGenOptions = {
     filename: 'bundle.json',
@@ -247,7 +256,7 @@ function parsePath(path?: string[]): string {
 
 export function generateMessageFunction(
   msg: string,
-  options: CodeGenOptions,
+  options: CodeGenOptions = {},
   path?: string[]
 ): CodeGenResult<ResourceNode> {
   const env = options.env != null ? options.env : 'development'
@@ -389,3 +398,58 @@ export function mapLinesColumns(
 
   return generator.toJSON()
 }
+
+export function generateResourceAst(
+  msg: string,
+  options: CodeGenOptions = {},
+  path?: string[]
+): CodeGenResult<ResourceNode> {
+  const env = options.env != null ? options.env : 'development'
+  const strictMessage = isBoolean(options.strictMessage)
+    ? options.strictMessage
+    : true
+  const escapeHtml = !!options.escapeHtml
+  const onError = options.onError || ON_ERROR_NOOP
+  const errors = [] as CompileError[]
+
+  let detecteHtmlInMsg = false
+  if (detectHtmlTag(msg)) {
+    detecteHtmlInMsg = true
+    if (strictMessage) {
+      const errMsg = format(DETECT_MESSAGE, { msg })
+      onError(format(errMsg), {
+        source: msg,
+        path: parsePath(path)
+      })
+    }
+  }
+
+  const _msg = detecteHtmlInMsg && escapeHtml ? sanitizeHtml(msg) : msg
+
+  const newOptions = Object.assign(
+    { location: env === 'development' },
+    options
+  ) as CompileOptions
+
+  if (newOptions.jit != null) {
+    newOptions.jit = true
+  }
+
+  newOptions.onError = (err: CompileError): void => {
+    if (onError) {
+      const extra: Parameters<Required<CodeGenOptions>['onError']>[1] = {
+        source: msg,
+        path: parsePath(path),
+        code: err.code,
+        domain: err.domain,
+        location: err.location
+      }
+      onError(err.message, extra)
+      errors.push(err)
+    }
+  }
+  const { ast, map } = baseCompile(_msg, newOptions)
+  const occured = errors.length > 0
+  const code = !occured ? `${friendlyJSONstringify(ast)}` : `\`${_msg}\``
+  return { code, ast, map, errors }
+}
diff --git a/packages/bundle-utils/src/js.ts b/packages/bundle-utils/src/js.ts
@@ -9,12 +9,18 @@ import { walk } from 'estree-walker'
 import {
   createCodeGenerator,
   generateMessageFunction,
+  generateResourceAst,
   mapLinesColumns
 } from './codegen'
 import { RawSourceMap } from 'source-map'
 
 import type { Node } from 'estree'
-import type { CodeGenOptions, CodeGenerator, CodeGenResult } from './codegen'
+import type {
+  CodeGenOptions,
+  CodeGenerator,
+  CodeGenResult,
+  CodeGenFunction
+} from './codegen'
 
 /**
  * @internal
@@ -36,7 +42,8 @@ export function generate(
     strictMessage = true,
     escapeHtml = false,
     useClassComponent = false,
-    allowDynamic = false
+    allowDynamic = false,
+    jit = false
   }: CodeGenOptions,
   injector?: () => string
 ): CodeGenResult<Node> {
@@ -60,7 +67,8 @@ export function generate(
     onError,
     strictMessage,
     escapeHtml,
-    useClassComponent
+    useClassComponent,
+    jit
   } as CodeGenOptions
   const generator = createCodeGenerator(options)
 
@@ -101,7 +109,7 @@ export function generate(
     }
   }
 
-  const codeMaps = generateNode(generator, ast, options, injector)
+  const codeMaps = _generate(generator, ast, options, injector)
 
   const { code, map } = generator.context()
   // if (map) {
@@ -144,10 +152,10 @@ function scanAst(ast: Node) {
   return ret
 }
 
-function generateNode(
+function _generate(
   generator: CodeGenerator,
   node: Node,
-  options: CodeGenOptions,
+  options: CodeGenOptions = {},
   injector?: () => string
 ): Map<string, RawSourceMap> {
   const propsCountStack = [] as number[]
@@ -163,9 +171,14 @@ function generateNode(
     sourceMap,
     isGlobal,
     locale,
-    useClassComponent
+    useClassComponent,
+    jit
   } = options
 
+  const codegenFn: CodeGenFunction = jit
+    ? generateResourceAst
+    : generateMessageFunction
+
   const componentNamespace = '_Component'
 
   walk(node, {
@@ -242,11 +255,7 @@ function generateNode(
                   const value = getValue(node.value) as string
                   generator.push(`${JSON.stringify(name)}: `)
                   pathStack.push(name)
-                  const { code, map } = generateMessageFunction(
-                    value,
-                    options,
-                    pathStack
-                  )
+                  const { code, map } = codegenFn(value, options, pathStack)
                   sourceMap && map != null && codeMaps.set(value, map)
                   generator.push(`${code}`, node.value, value)
                   skipStack.push(false)
@@ -256,7 +265,7 @@ function generateNode(
                     const strValue = JSON.stringify(value)
                     generator.push(`${JSON.stringify(name)}: `)
                     pathStack.push(name)
-                    const { code, map } = generateMessageFunction(
+                    const { code, map } = codegenFn(
                       strValue,
                       options,
                       pathStack
@@ -330,18 +339,14 @@ function generateNode(
                   node.type === 'TemplateLiteral'
                 ) {
                   const value = getValue(node) as string
-                  const { code, map } = generateMessageFunction(
-                    value,
-                    options,
-                    pathStack
-                  )
+                  const { code, map } = codegenFn(value, options, pathStack)
                   sourceMap && map != null && codeMaps.set(value, map)
                   generator.push(`${code}`, node, value)
                 } else {
                   const value = getValue(node)
                   if (forceStringify) {
                     const strValue = JSON.stringify(value)
-                    const { code, map } = generateMessageFunction(
+                    const { code, map } = codegenFn(
                       strValue,
                       options,
                       pathStack

diff --git a/packages/bundle-utils/src/json.ts b/packages/bundle-utils/src/json.ts
@@ -1,5 +1,5 @@
 /**
- * Code generator for i18n json/json5 resource
+ * Code / AST generator for i18n json/json5 resource
  */
 
 import {
@@ -11,18 +11,21 @@ import { isString, friendlyJSONstringify } from '@intlify/shared'
 import {
   createCodeGenerator,
   generateMessageFunction,
+  generateResourceAst,
   mapLinesColumns
 } from './codegen'
 import { generateLegacyCode } from './legacy'
 import { RawSourceMap } from 'source-map'
 import MagicString from 'magic-string'
 
 import type { JSONProgram, JSONNode } from 'jsonc-eslint-parser/lib/parser/ast'
-import type { CodeGenOptions, CodeGenerator, CodeGenResult } from './codegen'
+import type {
+  CodeGenOptions,
+  CodeGenerator,
+  CodeGenResult,
+  CodeGenFunction
+} from './codegen'
 
-/**
- * @internal
- */
 export function generate(
   targetSource: string | Buffer,
   {
@@ -40,7 +43,8 @@ export function generate(
     onError = undefined,
     strictMessage = true,
     escapeHtml = false,
-    useClassComponent = false
+    useClassComponent = false,
+    jit = false
   }: CodeGenOptions,
   injector?: () => string
 ): CodeGenResult<JSONProgram> {
@@ -67,7 +71,8 @@ export function generate(
     onError,
     strictMessage,
     escapeHtml,
-    useClassComponent
+    useClassComponent,
+    jit
   } as CodeGenOptions
   const generator = createCodeGenerator(options)
 
@@ -89,7 +94,7 @@ export function generate(
     }
   }
 
-  const codeMaps = generateNode(generator, ast, options, injector)
+  const codeMaps = _generate(generator, ast, options, injector)
 
   const { code, map } = generator.context()
   // if (map) {
@@ -99,7 +104,7 @@ export function generate(
   //   })
   // }
   // prettier-ignore
-  const newMap = map
+  const newMap = map && !jit
     ? mapLinesColumns((map as any).toJSON(), codeMaps, inSourceMap) || null // eslint-disable-line @typescript-eslint/no-explicit-any
     : null
   return {
@@ -109,10 +114,10 @@ export function generate(
   }
 }
 
-function generateNode(
+function _generate(
   generator: CodeGenerator,
   node: JSONProgram,
-  options: CodeGenOptions,
+  options: CodeGenOptions = {},
   injector?: () => string
 ): Map<string, RawSourceMap> {
   const propsCountStack = [] as number[]
@@ -127,9 +132,14 @@ function generateNode(
     sourceMap,
     isGlobal,
     locale,
-    useClassComponent
+    useClassComponent,
+    jit
   } = options
 
+  const codegenFn: CodeGenFunction = jit
+    ? generateResourceAst
+    : generateMessageFunction
+
   const componentNamespace = '_Component'
 
   traverseNodes(node, {
@@ -192,23 +202,15 @@ function generateNode(
             if (isString(value)) {
               generator.push(`${JSON.stringify(name)}: `)
               pathStack.push(name.toString())
-              const { code, map } = generateMessageFunction(
-                value,
-                options,
-                pathStack
-              )
+              const { code, map } = codegenFn(value, options, pathStack)
               sourceMap && map != null && codeMaps.set(value, map)
               generator.push(`${code}`, node.value, value)
             } else {
               if (forceStringify) {
                 const strValue = JSON.stringify(value)
                 generator.push(`${JSON.stringify(name)}: `)
                 pathStack.push(name.toString())
-                const { code, map } = generateMessageFunction(
-                  strValue,
-                  options,
-                  pathStack
-                )
+                const { code, map } = codegenFn(strValue, options, pathStack)
                 sourceMap && map != null && codeMaps.set(strValue, map)
                 generator.push(`${code}`, node.value, strValue)
               } else {
@@ -253,21 +255,13 @@ function generateNode(
             if (node.type === 'JSONLiteral') {
               const value = node.value
               if (isString(value)) {
-                const { code, map } = generateMessageFunction(
-                  value,
-                  options,
-                  pathStack
-                )
+                const { code, map } = codegenFn(value, options, pathStack)
                 sourceMap && map != null && codeMaps.set(value, map)
                 generator.push(`${code}`, node, value)
               } else {
                 if (forceStringify) {
                   const strValue = JSON.stringify(value)
-                  const { code, map } = generateMessageFunction(
-                    strValue,
-                    options,
-                    pathStack
-                  )
+                  const { code, map } = codegenFn(strValue, options, pathStack)
                   sourceMap && map != null && codeMaps.set(strValue, map)
                   generator.push(`${code}`, node, strValue)
                 } else {