fix decompression script etc

package.json

@@ -8,7 +8,8 @@
     "chess.js": "^1.0.0-beta.6",
     "d3": "^7.8.5",
     "node-zstandard": "^1.2.4",
-    "proper-lockfile": "^4.1.2"
+    "proper-lockfile": "^4.1.2",
+    "ts-node": "10.9.2"
   },
   "devDependencies": {
     "@babel/preset-typescript": "^7.23.0",

src/queue.ts

@@ -7,22 +7,25 @@ export const RESULTS_PATH = `${__dirname}/results.json`;
 function launchQueueServer() {
   // Create a write to result.json queue with a concurrency of 1
   // Possibly the simplest fix would be to run this as a separate process, then we can enforce messages sent to this queue are processed in order
-  const queue = asyncLib.queue<any>((task) => {
+  const queue = asyncLib.queue<any>((task, callback) => {
     console.log('received task', task.analysisKey);
-    return new Promise<void>((resolve, reject) => {
-      const { results, analysisKey } = task;
-      try {
-        fs.writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2));
-        console.log(
-          `Analysis "${analysisKey}" has been written to ${RESULTS_PATH}`
-        );
-        resolve();
-      } catch (err) {
-        reject(err);
-      }
-    });
+    // return new Promise<void>((resolve, reject) => {
+    const { results, analysisKey } = task;
+    try {
+      fs.writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2));
+      console.log(
+        `Analysis "${analysisKey}" has been written to ${RESULTS_PATH}`
+      );
+    } catch (err) {
+      console.error('Error writing to results.json', err);
+    }
+    // });
   }, 1);
 
+  queue.drain(function () {
+    console.log('all items have been processed');
+  });
+
   // this event listener receives tasks from the parallel processes
   const server = net.createServer((socket) => {
     socket.on('data', (data) => {

src/run_metrics_on_file.ts

@@ -24,20 +24,47 @@ import { RESULTS_PATH } from './queue';
  */
 export async function main(path: string) {
   console.time('Total Execution Time');
-  await gameIterator(path);
+  await gameIterator(path, { 'Number of games analyzed': 0 });
   console.timeEnd('Total Execution Time');
-  return results;
-}
 
-let results = {
-  'Number of games analyzed': 0,
-};
+  const now = new Date();
+  const milliseconds = now.getMilliseconds();
+
+  const analysisKey = `analysis_${now
+    .toLocaleString()
+    .replace(/\/|,|:|\s/g, '_')}_${milliseconds}`;
+
+  let existingResults = {};
+  if (fs.existsSync(RESULTS_PATH)) {
+    const fileContent = fs.readFileSync(RESULTS_PATH, 'utf8');
+    if (fileContent !== '') {
+      existingResults = JSON.parse(fileContent);
+    }
+  }
+
+  console.log('sending results');
+
+  // TODO: Probably we need to read in the existing results in the queue server and merge them, as when there are multiple items in the queue
+  // this is going to be out of date
+  existingResults[analysisKey] = {
+    'Number of games analyzed': 0,
+  };
+
+  const client = net.createConnection({ port: 8000 });
+
+  console.log('connected to queue server');
+
+  // Send the task to the queue server
+  client.write(JSON.stringify({ results: existingResults, analysisKey }));
+
+  console.log('results sent');
+}
 
 /**
  * Metric functions will ingest a single game at a time
  * @param metricFunctions
  */
-async function gameIterator(path) {
+async function gameIterator(path, results) {
   const cjsmin = new Chess();
 
   const gamesGenerator = gameChunks(path);
@@ -74,27 +101,5 @@ async function gameIterator(path) {
 
 // for use with zst_decompresser.js
 if (require.main === module) {
-  main(process.argv[2]).then((results) => {
-    const now = new Date();
-    const milliseconds = now.getMilliseconds();
-
-    const analysisKey = `analysis_${now
-      .toLocaleString()
-      .replace(/\/|,|:|\s/g, '_')}_${milliseconds}`;
-
-    let existingResults = {};
-    if (fs.existsSync(RESULTS_PATH)) {
-      const fileContent = fs.readFileSync(RESULTS_PATH, 'utf8');
-      if (fileContent !== '') {
-        existingResults = JSON.parse(fileContent);
-      }
-    }
-
-    existingResults[analysisKey] = results;
-
-    const client = net.createConnection({ port: 8000 });
-
-    // Send the task to the queue server
-    client.write(JSON.stringify({ results: existingResults, analysisKey }));
-  });
+  main(process.argv[2]).then((results) => {});
 }