Feature: in-memory cache file CRC32 calculation (#430)

src/constants.ts

@@ -97,6 +97,11 @@ export default class Constants {
    */
   static readonly OUTPUT_CLEANER_BATCH_SIZE = 100;
 
+  /**
+   * Number of file checksums to cache in memory at once.
+   */
+  static readonly FILE_CHECKSUM_CACHE_SIZE = 128;
+
   /**
    * Max {@link fs} highWaterMark chunk size to read and write at a time.
    */

src/types/cache.ts

@@ -0,0 +1,52 @@
+import { Mutex } from 'async-mutex';
+
+export default class Cache<K, V> {
+  private readonly keyOrder: K[] = [];
+
+  private readonly keyValues = new Map<K, V>();
+
+  private readonly keyMutexes = new Map<K, Mutex>();
+
+  private readonly keyMutexesMutex = new Mutex();
+
+  private readonly maxSize: number;
+
+  constructor(maxSize: number) {
+    this.maxSize = maxSize;
+  }
+
+  public async getOrCompute(key: K, runnable: () => (V | Promise<V>)): Promise<V> {
+    // Get a mutex for `key`
+    const keyMutex = await this.keyMutexesMutex.runExclusive(() => {
+      if (!this.keyMutexes.has(key)) {
+        this.keyMutexes.set(key, new Mutex());
+      }
+      return this.keyMutexes.get(key) as Mutex;
+    });
+
+    // Only allow one concurrent fetch/compute for `key`
+    return keyMutex.runExclusive(async () => {
+      if (this.keyValues.has(key)) {
+        return this.keyValues.get(key) as V;
+      }
+
+      const val = await runnable();
+      this.set(key, val);
+      return val;
+    });
+  }
+
+  private set(key: K, val: V): void {
+    if (!this.keyValues.has(key)) {
+      this.keyOrder.push(key);
+    }
+    this.keyValues.set(key, val);
+
+    // Delete old values
+    if (this.keyValues.size > this.maxSize) {
+      const staleKey = this.keyOrder.splice(0, 1)[0];
+      this.keyValues.delete(staleKey);
+      this.keyMutexes.delete(staleKey);
+    }
+  }
+}

src/types/files/file.ts

@@ -9,10 +9,15 @@ import Constants from '../../constants.js';
 import FilePoly from '../../polyfill/filePoly.js';
 import fsPoly from '../../polyfill/fsPoly.js';
 import URLPoly from '../../polyfill/urlPoly.js';
+import Cache from '../cache.js';
 import Patch from '../patches/patch.js';
 import ROMHeader from './romHeader.js';
 
 export default class File {
+  private static readonly crc32Cache = new Cache<string, string>(
+    Constants.FILE_CHECKSUM_CACHE_SIZE,
+  );
+
   private readonly filePath: string;
 
   private readonly size: number;
@@ -141,7 +146,8 @@ export default class File {
   ): Promise<string> {
     const start = fileHeader?.getDataOffsetBytes() || 0;
 
-    return new Promise((resolve, reject) => {
+    const cacheKey = `${localFile}|${start}`;
+    return File.crc32Cache.getOrCompute(cacheKey, async () => new Promise((resolve, reject) => {
       const stream = fs.createReadStream(localFile, {
         start,
         highWaterMark: Constants.FILE_READING_CHUNK_SIZE,
@@ -160,7 +166,7 @@ export default class File {
       });
 
       stream.on('error', reject);
-    });
+    }));
   }
 
   async extractToFile(destinationPath: string): Promise<void> {