fix: Optimize memory usage when parsing LgFiles

Composer/packages/client/src/App.tsx

@@ -2,17 +2,20 @@
 // Licensed under the MIT License.
 
 import React, { Fragment, useEffect, useState } from 'react';
-import { useRecoilValue } from 'recoil';
+import { useRecoilValue, useRecoilCallback, CallbackInterface } from 'recoil';
+import { useMount, useUnmount } from '@fluentui/react-hooks';
 
 import { Header } from './components/Header';
 import { Announcement } from './components/AppComponents/Announcement';
 import { MainContainer } from './components/AppComponents/MainContainer';
-import { dispatcherState, userSettingsState } from './recoilModel';
+import { dispatcherState, userSettingsState, lgFileState } from './recoilModel';
 import { loadLocale } from './utils/fileUtil';
 import { useInitializeLogger } from './telemetry/useInitializeLogger';
 import { setupIcons } from './setupIcons';
 import { setOneAuthEnabled } from './utils/oneAuthUtil';
 import { LoadingSpinner } from './components/LoadingSpinner';
+import lgWorker from './recoilModel/parsers/lgWorker';
+import { LgEventType } from './recoilModel/parsers/types';
 
 setupIcons();
 
@@ -26,6 +29,7 @@ export const App: React.FC = () => {
   const { appLocale } = useRecoilValue(userSettingsState);
 
   const [isClosing, setIsClosing] = useState(false);
+  const [listener, setListener] = useState<{ destroy(): boolean }>({} as any);
 
   const {
     fetchExtensions,
@@ -34,6 +38,19 @@ export const App: React.FC = () => {
     performAppCleanupOnQuit,
     setMachineInfo,
   } = useRecoilValue(dispatcherState);
+  const updateFile = useRecoilCallback((callbackHelpers: CallbackInterface) => async ({ projectId, value }) => {
+    callbackHelpers.set(lgFileState({ projectId, lgFileId: value.id }), value);
+  });
+
+  useMount(() => {
+    const listener = lgWorker.listen(LgEventType.OnUpdateLgFile, (msg) => {
+      const { projectId, payload } = msg.data;
+      updateFile({ projectId, value: payload });
+    });
+    setListener(listener);
+  });
+
+  useUnmount(() => listener.destroy());
 
   useEffect(() => {
     loadLocale(appLocale);

Composer/packages/client/src/recoilModel/parsers/lgWorker.ts

@@ -6,6 +6,7 @@ import Worker from './workers/lgParser.worker.ts';
 import { BaseWorker } from './baseWorker';
 import {
   LgActionType,
+  LgEventType,
   LgParsePayload,
   LgUpdateTemplatePayload,
   LgCreateTemplatePayload,
@@ -20,6 +21,39 @@ import {
 
 // Wrapper class
 class LgWorker extends BaseWorker<LgActionType> {
+  private listeners = new Map<LgEventType, ((msg: MessageEvent) => void)[]>();
+
+  constructor(worker: Worker) {
+    super(worker);
+
+    worker.onmessage = (msg) => {
+      const { type } = msg.data;
+
+      if (type === LgEventType.OnUpdateLgFile) {
+        this.listeners.get(type)?.forEach((cb) => cb(msg));
+      } else {
+        this.handleMsg(msg);
+      }
+    };
+  }
+
+  listen(action: LgEventType, callback: (msg: MessageEvent) => void) {
+    if (this.listeners.has(action)) {
+      this.listeners.get(action)!.push(callback);
+    } else {
+      this.listeners.set(action, [callback]);
+    }
+
+    return {
+      destroy: () => this.listeners.delete(action),
+    };
+  }
+
+  flush(): Promise<boolean> {
+    this.listeners.clear();
+    return super.flush();
+  }
+
   addProject(projectId: string) {
     return this.sendMsg<LgNewCachePayload>(LgActionType.NewCache, { projectId });
   }

Composer/packages/client/src/recoilModel/parsers/types.ts

@@ -156,6 +156,10 @@ export enum LgActionType {
   ParseAll = 'parse-all',
 }
 
+export enum LgEventType {
+  OnUpdateLgFile = 'on-update-lgfile',
+}
+
 export enum IndexerActionType {
   Index = 'index',
 }

Composer/packages/client/src/recoilModel/parsers/workers/lgParser.worker.ts

@@ -5,6 +5,7 @@ import { lgImportResolverGenerator, LgFile } from '@bfc/shared';
 
 import {
   LgActionType,
+  LgEventType,
   LgParsePayload,
   LgUpdateTemplatePayload,
   LgCreateTemplatePayload,
@@ -16,6 +17,7 @@ import {
   LgCleanCachePayload,
   LgParseAllPayload,
 } from '../types';
+import { MapOptimizer } from '../../utils/mapOptimizer';
 
 const ctx: Worker = self as any;
 
@@ -197,6 +199,11 @@ export const handleMessage = (msg: LgMessageEvent) => {
     case LgActionType.Parse: {
       const { id, content, lgFiles, projectId } = msg.payload;
 
+      const cachedFile = cache.get(projectId, id);
+      if (cachedFile?.isContentUnparsed === false && cachedFile?.content === content) {
+        return filterParseResult(cachedFile);
+      }
+
       const lgFile = lgUtil.parse(id, content, lgFiles);
       cache.set(projectId, lgFile);
       payload = filterParseResult(lgFile);
@@ -206,12 +213,20 @@ export const handleMessage = (msg: LgMessageEvent) => {
     case LgActionType.ParseAll: {
       const { lgResources, projectId } = msg.payload;
       // We'll do the parsing when the file is required. Save empty LG instead.
-      payload = lgResources.map(({ id, content }) => {
-        const emptyLg = emptyLgFile(id, content);
-        cache.set(projectId, emptyLg);
-        return filterParseResult(emptyLg);
+      payload = lgResources.map(({ id, content }) => [id, emptyLgFile(id, content)]);
+      const resources = new Map<string, LgFile>(payload);
+      cache.projects.set(projectId, resources);
+
+      const optimizer = new MapOptimizer(10, resources);
+      optimizer.onUpdate((_, value, ctx) => {
+        const refs = value.parseResult?.references?.map(({ name }) => name);
+        ctx.setReferences(refs);
+      });
+      optimizer.onDelete((_, value) => {
+        const lgFile = emptyLgFile(value.id, value.content);
+        cache.set(projectId, lgFile);
+        ctx.postMessage({ type: LgEventType.OnUpdateLgFile, projectId, payload: lgFile });
       });
-
       break;
     }
 

Composer/packages/client/src/recoilModel/utils/mapOptimizer.ts

@@ -0,0 +1,141 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/**
+ * Internal tree structure to track the oldest elements and their references.
+ */
+interface MapOptimizerTree<Key> {
+  timestamp: number;
+  references: Key[];
+}
+
+/**
+ * Context for the MapOptimizer.onUpdate event.
+ */
+interface OnUpdateMapOptimizerContext<Key> {
+  /**
+   * Sets the related Map keys references of an element, these references are taken into account on the delete event.
+   * @param references The Map keys of a related element.
+   */
+  setReferences(references: Key[]): void;
+}
+
+/**
+ * Class to optimize a Map object by deleting the oldest elements of the collection based on a capacity limit.
+ */
+export class MapOptimizer<Key, Value> {
+  public tree = new Map<Key, MapOptimizerTree<Key>>();
+  private skipOptimize = new Set<Key>();
+
+  onUpdateCallback?: (key: Key, value: Value, ctx: OnUpdateMapOptimizerContext<Key>) => void;
+  onDeleteCallback?: (key: Key, value: Value) => void;
+
+  /**
+   * Initializes a new instance of the MapOptimizer class.
+   * @param capacity The capacity limit to trigger the optimization steps.
+   * @param list The Map object to optimize.
+   */
+  constructor(private capacity: number, public list: Map<Key, Value>) {
+    this.attach();
+  }
+
+  /**
+   * Event triggered when an element is added or updated in the Map object.
+   * @param callback Exposes the element's Key, Value and Context to perform operations.
+   */
+  onUpdate(callback: (key: Key, value: Value, ctx: OnUpdateMapOptimizerContext<Key>) => void) {
+    this.onUpdateCallback = callback;
+  }
+
+  /**
+   * Event triggered when an element is marked for deletion.
+   * @param callback Exposes the element's Key, Value.
+   */
+  onDelete(callback: (key: Key, value: Value) => void) {
+    this.onDeleteCallback = callback;
+  }
+
+  /**
+   * @private
+   * Attaches the "set" method to the Map object to listen and trigger the optimization.
+   */
+  private attach() {
+    const set = this.list.set;
+    this.list.set = (key, value) => {
+      if (!this.skipOptimize.has(key)) {
+        this.optimize(key, value);
+      }
+      const result = set.apply(this.list, [key, value]);
+      return result;
+    };
+  }
+
+  /**
+   * @private
+   * Optimizes the Map object by performing the onDelete event callback on the oldest element in the collection.
+   */
+  private optimize(keyToAdd: Key, valueToAdd: Value) {
+    const exists = this.tree.has(keyToAdd);
+    const context: MapOptimizerTree<Key> = { timestamp: Date.now(), references: [] };
+    this.onUpdateCallback?.(keyToAdd, valueToAdd, {
+      setReferences: (references) => (context.references = references || []),
+    });
+    this.tree.set(keyToAdd, context);
+
+    if (exists) {
+      return;
+    }
+
+    let processed: [Key, MapOptimizerTree<Key>][] = [];
+    const itemsToRemove = Array.from(this.tree.entries())
+      .filter(([key]) => key !== keyToAdd)
+      .sort(([, v1], [, v2]) => v2.timestamp - v1.timestamp);
+
+    while (this.capacity < this.tree.size) {
+      const itemToRemove = itemsToRemove.pop();
+      if (!itemToRemove) {
+        break;
+      }
+
+      const [key, { references }] = itemToRemove;
+      const ids = this.identify([key, ...references]);
+
+      // Re-process previous items if an item gets deleted.
+      processed.push(itemToRemove);
+      if (ids.length > 0) {
+        itemsToRemove.push(...processed);
+        processed = [];
+      }
+
+      for (const id of ids) {
+        this.tree.delete(id);
+        const listItem = this.list.get(id)!;
+        this.skipOptimize.add(id);
+        this.onDeleteCallback ? this.onDeleteCallback(id, listItem) : this.list.delete(id);
+        this.skipOptimize.delete(id);
+      }
+    }
+  }
+
+  /**
+   * @private
+   * Identifies all the keys that are available to delete.
+   */
+  private identify(references: Key[], memo: Key[] = []) {
+    for (const reference of references) {
+      const found = this.tree.get(reference);
+      const existsOnMemo = () => memo.some((e) => found!.references.includes(e));
+      const existsOnReferences = () =>
+        Array.from(this.tree.values()).some(({ references }) => references.includes(reference));
+
+      if (!found || existsOnMemo() || existsOnReferences()) {
+        continue;
+      }
+
+      memo.push(reference);
+      this.identify(found.references, memo);
+    }
+
+    return memo;
+  }
+}