export-res-path-for-consistency

src/queue.ts

@@ -0,0 +1,41 @@
+import * as asyncLib from 'async';
+import * as fs from 'fs';
+import * as net from 'net';
+// will just write to wherever the process is running, but the server needs to be launched from the same directory so we use an abs path
+export const RESULTS_PATH = `${__dirname}/results.json`;
+
+function launchQueueServer() {
+  // Create a write to result.json queue with a concurrency of 1
+  // Possibly the simplest fix would be to run this as a separate process, then we can enforce messages sent to this queue are processed in order
+  const queue = asyncLib.queue<any>((task) => {
+    console.log('received task', task.analysisKey);
+    return new Promise<void>((resolve, reject) => {
+      const { results, analysisKey } = task;
+      try {
+        fs.writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2));
+        console.log(
+          `Analysis "${analysisKey}" has been written to ${RESULTS_PATH}`
+        );
+        resolve();
+      } catch (err) {
+        reject(err);
+      }
+    });
+  }, 1);
+
+  // this event listener receives tasks from the parallel processes
+  const server = net.createServer((socket) => {
+    socket.on('data', (data) => {
+      const task = JSON.parse(data.toString());
+      queue.push(task);
+    });
+  });
+
+  console.log('Queue server listening on port 8000');
+  server.listen(8000);
+}
+
+// for use with zst_decompresser.js
+if (require.main === module) {
+  launchQueueServer();
+}

src/run_metrics_on_file.ts

@@ -0,0 +1,100 @@
+import * as fs from 'fs';
+import * as net from 'net';
+import { Chess } from '../cjsmin/src/chess';
+import { gameChunks } from './fileReader';
+import {
+  KDRatioMetric,
+  KillStreakMetric,
+  MateAndAssistMetric,
+} from './metrics/captures';
+import { MoveDistanceMetric } from './metrics/distances';
+import { MetadataMetric } from './metrics/misc';
+import {
+  GameWithMostMovesMetric,
+  MiscMoveFactMetric,
+  PieceLevelMoveInfoMetric,
+} from './metrics/moves';
+import { PromotionMetric } from './metrics/promotions';
+import { RESULTS_PATH } from './queue';
+
+/**
+ *
+ * @param path
+ * @returns
+ */
+export async function main(path: string) {
+  console.time('Total Execution Time');
+  await gameIterator(path);
+  console.timeEnd('Total Execution Time');
+  return results;
+}
+
+let results = {
+  'Number of games analyzed': 0,
+};
+
+/**
+ * Metric functions will ingest a single game at a time
+ * @param metricFunctions
+ */
+async function gameIterator(path) {
+  const cjsmin = new Chess();
+
+  const gamesGenerator = gameChunks(path);
+
+  const metrics = [
+    new KDRatioMetric(),
+    new KillStreakMetric(),
+    new MateAndAssistMetric(),
+    new PromotionMetric(),
+    new MoveDistanceMetric(),
+    new GameWithMostMovesMetric(),
+    new PieceLevelMoveInfoMetric(),
+    new MetadataMetric(cjsmin),
+    new MiscMoveFactMetric(),
+  ];
+
+  let gameCounter = 0;
+  for await (const { moves, metadata } of gamesGenerator) {
+    if (gameCounter++ % 400 == 0) console.log(`ingested ${gameCounter} games`);
+
+    for (const metric of metrics) {
+      // with array creation
+      const historyGenerator = cjsmin.historyGeneratorArr(moves);
+      // the generator is useless if we convert it to an array
+      metric.processGame(Array.from(historyGenerator), metadata);
+    }
+  }
+
+  results['Number of games analyzed'] = gameCounter;
+  for (const metric of metrics) {
+    results[metric.constructor.name] = metric.aggregate();
+  }
+}
+
+// for use with zst_decompresser.js
+if (require.main === module) {
+  main(process.argv[2]).then((results) => {
+    const now = new Date();
+    const milliseconds = now.getMilliseconds();
+
+    const analysisKey = `analysis_${now
+      .toLocaleString()
+      .replace(/\/|,|:|\s/g, '_')}_${milliseconds}`;
+
+    let existingResults = {};
+    if (fs.existsSync(RESULTS_PATH)) {
+      const fileContent = fs.readFileSync(RESULTS_PATH, 'utf8');
+      if (fileContent !== '') {
+        existingResults = JSON.parse(fileContent);
+      }
+    }
+
+    existingResults[analysisKey] = results;
+
+    const client = net.createConnection({ port: 8000 });
+
+    // Send the task to the queue server
+    client.write(JSON.stringify({ results: existingResults, analysisKey }));
+  });
+}