Improve ShaderLab compilation error log and package build

.github/workflows/ci.yml

@@ -50,7 +50,7 @@ jobs:
 
       - name: Install
         run: pnpm install
-      - run: npm run build:editor
+      - run: npm run build
 
   codecov:
     runs-on: macos-latest
@@ -66,7 +66,7 @@ jobs:
       - name: Install
         run: pnpm install
       - name: Build
-        run: npm run build:editor
+        run: npm run build
       - name: Test
         run: npm run test-cov
       - run: pnpm install codecov -w
@@ -94,7 +94,7 @@ jobs:
       - name: Run Cypress Tests
         uses: cypress-io/github-action@v5
         with:
-          build: npm run build:editor
+          build: npm run build
           start: npm run e2e:case
           wait-on: "http://localhost:5175"
           wait-on-timeout: 120

package.json

@@ -9,19 +9,14 @@
     "test-debug": "cross-env TS_NODE_PROJECT=tsconfig.tests.json floss --path tests -r ts-node/register --debug",
     "test-cov": "cross-env TS_NODE_PROJECT=tsconfig.tests.json IS_COV=1 nyc --reporter=lcov floss --path tests -r ts-node/register",
     "build": "npm run b:module && npm run b:types",
-    "build:editor": "npm run b:module:editor && npm run b:types",
     "lint": "eslint packages/*/src --ext .ts",
     "watch": "cross-env NODE_ENV=release BUILD_TYPE=MODULE rollup -cw -m inline",
-    "watch:editor": "cross-env NODE_ENV=editor BUILD_TYPE=MODULE rollup -cw -m inline",
     "watch:umd": "cross-env NODE_ENV=release BUILD_TYPE=UMD rollup -cw -m inline",
-    "watch:umd:editor": "cross-env NODE_ENV=editor BUILD_TYPE=UMD rollup -cw -m inline",
     "b:types": "pnpm -r --filter=./packages/* run b:types",
     "b:module": "cross-env BUILD_TYPE=MODULE NODE_ENV=release rollup -c",
-    "b:module:editor": "cross-env BUILD_TYPE=MODULE NODE_ENV=editor rollup -c",
     "b:umd": "cross-env BUILD_TYPE=UMD NODE_ENV=release rollup -c",
     "b:miniprogram": "cross-env BUILD_TYPE=MINI rollup -c",
     "b:all": "cross-env NODE_ENV=release npm run b:types && cross-env BUILD_TYPE=ALL NODE_ENV=release rollup -c",
-    "b:all:editor": "cross-env NODE_ENV=editor npm run b:types && cross-env BUILD_TYPE=ALL rollup -c",
     "clean": "pnpm -r exec rm -rf dist && pnpm -r exec rm -rf types",
     "e2e:case": "pnpm -C ./e2e run case",
     "e2e": "cypress run --browser chrome --headless",

packages/shader-lab/README.md

@@ -26,6 +26,14 @@ const shader = Shader.create(galaceanShaderCode);
 engine.run()
 ```
 
+There are two versions of ShaderLab: `Release` and `Verbose`. The `Verbose` version offers more user-friendly diagnostic information for debug ShaderLab compilation errors, while the Release version provides superior performance.
+
+you can use `Verbose` version by import:
+
+```ts
+import { ShaderLab } from "@galacean/engine-shader-lab/verbose";
+```
+
 ## CFG Grammar conflict detection
 
 The Galacean ShaderLab syntax is defined using Context-Free Grammar (CFG) and is documented within the `\*.y` file. When modifications to the ShaderLab syntax are required, it is recommended to make changes to the existing CFG syntax file, and employ [Bison](https://www.gnu.org/software/bison/manual/bison.html) to detect any potential grammar conflicts.

packages/shader-lab/package.json

@@ -25,7 +25,8 @@
   },
   "files": [
     "dist/**/*",
-    "types/**/*"
+    "types/**/*",
+    "verbose/package.json"
   ],
   "devDependencies": {
     "@galacean/engine-design": "workspace:*",

packages/shader-lab/src/GSError.ts

@@ -0,0 +1,65 @@
+// #if _VERBOSE
+import { ShaderPosition } from "./common/ShaderPosition";
+import { ShaderRange } from "./common/ShaderRange";
+
+export class GSError extends Error {
+  static wrappingLineCount = 2;
+
+  constructor(
+    name: GSErrorName,
+    message: string,
+    public readonly location: ShaderRange | ShaderPosition,
+    public readonly source: string,
+    public readonly file?: string
+  ) {
+    super(message);
+    this.name = name;
+  }
+
+  override toString(): string {
+    let start: ShaderPosition, end: ShaderPosition;
+    const { message, location, source } = this;
+    if (!source) {
+      return message;
+    }
+
+    if (location instanceof ShaderPosition) {
+      start = end = location;
+    } else {
+      start = location.start;
+      end = location.end;
+    }
+    const lines = source.split("\n");
+
+    let diagnosticMessage = `${this.name}: ${message}\n\n`;
+    const lineSplit = "|···";
+
+    for (let i = start.line - GSError.wrappingLineCount, n = end.line + GSError.wrappingLineCount; i <= n; i++) {
+      diagnosticMessage += lineSplit + `${lines[i]}\n`;
+
+      if (i < start.line || i > end.line) continue;
+
+      let remarkStart = 0;
+      let remarkEnd = lines[i].length;
+      let paddingLength = lineSplit.length;
+      if (i === start.line) {
+        remarkStart = start.column;
+        paddingLength += start.column;
+      } else if (i === end.line) {
+        remarkEnd = end.column;
+      }
+      const remarkLength = Math.max(remarkEnd - remarkStart, 1);
+
+      diagnosticMessage += " ".repeat(paddingLength) + "^".repeat(remarkLength) + "\n";
+    }
+
+    return diagnosticMessage;
+  }
+}
+
+// #endif
+export enum GSErrorName {
+  PreprocessorError = "PreprocessorError",
+  CompilationError = "CompilationError",
+  ScannerError = "ScannerError"
+}

packages/shader-lab/src/Utils.ts → packages/shader-lab/src/ParserUtils.ts

@@ -1,11 +1,10 @@
 import { ENonTerminal, GrammarSymbol } from "./parser/GrammarSymbol";
 import { BaseToken as Token } from "./common/BaseToken";
-import { EKeyword, ETokenType, GalaceanDataType, ShaderRange, ShaderPosition } from "./common";
+import { EKeyword, ETokenType, GalaceanDataType } from "./common";
 import { TreeNode } from "./parser/AST";
-// #if _EDITOR
+// #if _VERBOSE
 import State from "./lalr/State";
 // #endif
-import { Logger } from "@galacean/engine";
 
 export class ParserUtils {
   static unwrapNodeByType<T = TreeNode>(node: TreeNode, type: ENonTerminal): T | undefined {
@@ -15,7 +14,7 @@ export class ParserUtils {
     return ParserUtils.unwrapNodeByType(child, type);
   }
 
-  // #if _EDITOR
+  // #if _VERBOSE
   /**
    * Check if type `tb` is compatible with type `ta`.
    */
@@ -39,15 +38,10 @@ export class ParserUtils {
     return sm < ENonTerminal.START;
   }
 
-  static throw(pos: ShaderPosition | ShaderRange | number, ...msgs: any[]) {
-    Logger.error(pos.toString(), ...msgs);
-    throw msgs.join(" ");
-  }
-
   /**
    * @internal
    */
-  // #if _EDITOR
+  // #if _VERBOSE
   static printStatePool(logPath: string) {
     let output = "";
 

packages/shader-lab/src/ShaderLab.ts

@@ -2,38 +2,32 @@
 import { ShaderTargetParser } from "./parser";
 import { Preprocessor } from "./preprocessor";
 import { GLES100Visitor, GLES300Visitor } from "./codeGen";
-import { IShaderContent, IShaderLab } from "@galacean/engine-design/src/shader-lab";
+import { IShaderContent, IShaderLab } from "@galacean/engine-design";
 import { ShaderContentParser } from "./contentParser";
 // @ts-ignore
-import { Logger, ShaderLib, ShaderMacro, ShaderPass, ShaderPlatformTarget } from "@galacean/engine";
+import { Logger, ShaderLib, ShaderMacro, ShaderPlatformTarget } from "@galacean/engine";
 import { ShaderPosition, ShaderRange } from "./common";
-import { ShaderLabObjectPool } from "./ShaderLabObjectPool";
+// #if _VERBOSE
+import { GSError } from "./GSError";
+// #endif
+import { PpParser } from "./preprocessor/PpParser";
+import { ShaderLabUtils } from "./ShaderLabUtils";
 
+/** @internal */
 export class ShaderLab implements IShaderLab {
-  /**
-   * @internal
-   */
   private static _parser = ShaderTargetParser.create();
-  /**
-   * @internal
-   */
-  private static _shaderPositionPool = new ShaderLabObjectPool(ShaderPosition);
-  /**
-   * @internal
-   */
-  private static _shaderRangePool = new ShaderLabObjectPool(ShaderRange);
+  private static _shaderPositionPool = ShaderLabUtils.createObjectPool(ShaderPosition);
+  private static _shaderRangePool = ShaderLabUtils.createObjectPool(ShaderRange);
 
-  static createPosition(
-    index: number,
-    // #if _EDITOR
-    line?: number,
-    column?: number
-    // #endif
-  ): ShaderPosition {
+  // #if _VERBOSE
+  static _processingPassText?: string;
+  // #endif
+
+  static createPosition(index: number, line?: number, column?: number): ShaderPosition {
     const position = this._shaderPositionPool.get();
-    position.setX(
+    position.set(
       index,
-      // #if _EDITOR
+      // #if _VERBOSE
       line,
       column
       // #endif
@@ -43,10 +37,15 @@
 
   static createRange(start: ShaderPosition, end: ShaderPosition): ShaderRange {
     const range = this._shaderRangePool.get();
-    range.setX(start, end);
+    range.set(start, end);
     return range;
   }
 
+  // #if _VERBOSE
+  /** Retrieve the compilation errors */
+  readonly errors: GSError[] = [];
+  // #endif
+
   _parseShaderPass(
     source: string,
     vertexEntry: string,
@@ -56,7 +55,6 @@
     platformMacros: string[],
     basePathForIncludeKey: string
   ) {
-    ShaderLabObjectPool.clearAllShaderLabObjectPool();
     Preprocessor.reset(ShaderLib, basePathForIncludeKey);
     for (const macro of macros) {
       Preprocessor.addPredefinedMacro(macro.name, macro.value);
@@ -66,68 +64,82 @@
       Preprocessor.addPredefinedMacro(platformMacros[i]);
     }
 
-    // #if _EDITOR
-    // TODO: index to position
-    // Logger.convertSourceIndex = Preprocessor.convertSourceIndex.bind(Preprocessor);
-    // #endif
-
     const preprocessorStart = performance.now();
-
     const ppdContent = Preprocessor.process(source);
+    // #if _VERBOSE
+    if (PpParser._errors.length > 0) {
+      for (const err of PpParser._errors) {
+        this.errors.push(<GSError>err);
+      }
+      this._logErrors();
+      return { vertex: "", fragment: "" };
+    }
+    // #endif
 
     Logger.info(`[pass compilation - preprocessor]  cost time ${performance.now() - preprocessorStart}ms`);
 
     const lexer = new Lexer(ppdContent);
     const tokens = lexer.tokenize();
-    const program = ShaderLab._parser.parse(tokens);
+
+    const { _parser: parser } = ShaderLab;
+
+    ShaderLab._processingPassText = ppdContent;
+    const program = parser.parse(tokens);
+
+    // #if _VERBOSE
+    for (const err of parser.errors) {
+      this.errors.push(err);
+    }
     if (!program) {
+      this._logErrors();
       return { vertex: "", fragment: "" };
     }
+    // #endif
+
     const codeGen =
       backend === ShaderPlatformTarget.GLES100 ? GLES100Visitor.getVisitor() : GLES300Visitor.getVisitor();
 
     const start = performance.now();
     const ret = codeGen.visitShaderProgram(program, vertexEntry, fragmentEntry);
     Logger.info(`[CodeGen] cost time: ${performance.now() - start}ms`);
+    ShaderLab._processingPassText = undefined;
+
+    // #if _VERBOSE
+    for (const err of codeGen.errors) {
+      this.errors.push(err);
+    }
+    this._logErrors();
+    // #endif
 
     return ret;
   }
 
   _parseShaderContent(shaderSource: string): IShaderContent {
+    ShaderLabUtils.clearAllShaderLabObjectPool();
     ShaderContentParser.reset();
-    return ShaderContentParser.parse(shaderSource);
+    const ret = ShaderContentParser.parse(shaderSource);
+
+    // #if _VERBOSE
+    this.errors.length = 0;
+    for (const error of ShaderContentParser._errors) {
+      this.errors.push(error);
+    }
+    // #endif
+
+    return ret;
   }
 
-  // #if _EDITOR
+  // #if _VERBOSE
   /**
    * @internal
-   * For debug
    */
-  _parse(
-    shaderSource: string,
-    macros: ShaderMacro[],
-    backend: ShaderPlatformTarget
-  ): (ReturnType<ShaderLab["_parseShaderPass"]> & { name: string })[] {
-    const structInfo = this._parseShaderContent(shaderSource);
-    const passResult = [] as any;
-    for (const subShader of structInfo.subShaders) {
-      for (const pass of subShader.passes) {
-        if (pass.isUsePass) continue;
-        const passInfo = this._parseShaderPass(
-          pass.contents,
-          pass.vertexEntry,
-          pass.fragmentEntry,
-          macros,
-          backend,
-          [],
-          // @ts-ignore
-          new URL(pass.name, ShaderPass._shaderRootPath).href
-        ) as any;
-        passInfo.name = pass.name;
-        passResult.push(passInfo);
-      }
+  _logErrors() {
+    const errors = this.errors;
+    if (errors.length === 0 || !Logger.isEnabled) return;
+    Logger.error(`${errors.length} errors occur!`);
+    for (const err of errors) {
+      Logger.error(err.toString());
     }
-    return passResult;
   }
   // #endif
 }