From b0b3eb842e15c30ef3f109d04687ad39e39e2531 Mon Sep 17 00:00:00 2001 From: EllAchE <26192612+EllAchE@users.noreply.github.com> Date: Mon, 11 Mar 2024 01:00:52 -0700 Subject: [PATCH] only run functions if the entry point is the specified module --- src/aggregate_analysis.ts | 411 +++++++++++++++++++++++++++++--------- src/zst_decompressor.ts | 27 +-- 2 files changed, 329 insertions(+), 109 deletions(-) diff --git a/src/aggregate_analysis.ts b/src/aggregate_analysis.ts index b450c64..66a954f 100644 --- a/src/aggregate_analysis.ts +++ b/src/aggregate_analysis.ts @@ -117,16 +117,11 @@ async function aggregateResults(filePath: string) { )); // ratings largest diff - const thisLargestRatingDiff = - analysis['MetadataMetric']['largestRatingDiff']; - const thisLargestRatingDiffGame = - analysis['MetadataMetric']['largestRatingDiffGame']; - if (thisLargestRatingDiff > largestRatingDiff) { - largestRatingDiff = thisLargestRatingDiff; - largestRatingDiffGame = thisLargestRatingDiffGame; - } else if (thisLargestRatingDiff === largestRatingDiff) { - largestRatingDiffGame.push(thisLargestRatingDiffGame); - } + ({ largestRatingDiff, largestRatingDiffGame } = aggregateRatingDiff( + analysis, + largestRatingDiff, + largestRatingDiffGame + )); // games played // currently this stat is inaccurately tracked across analyses @@ -226,41 +221,20 @@ async function aggregateResults(filePath: string) { } // piece level moves metrics - const thisTotalMovesByPiece = - analysis['PieceLevelMoveInfoMetric']['totalMovesByPiece']; - for (const uas in thisTotalMovesByPiece) { - if (!totalMovesByPiece[uas]) { - totalMovesByPiece[uas] = { - numMoves: thisTotalMovesByPiece[uas].numMoves, - }; - } - totalMovesByPiece[uas].numMoves += thisTotalMovesByPiece[uas].numMoves; - } - - const thisSingleGameMaxMoves = - analysis['PieceLevelMoveInfoMetric']['uasSingleGameMaxMoves']; - const thisPieceSingleGameMaxMoves = - analysis['PieceLevelMoveInfoMetric']['uasWithMostMovesSingleGame']; - const thisGameSingleGameMaxMoves = - analysis['PieceLevelMoveInfoMetric']['gamesWithUasMostMoves']; - if (thisSingleGameMaxMoves > singleGameMaxMoves) { - singleGameMaxMoves = thisSingleGameMaxMoves; - pieceSingleGameMaxMoves = [thisPieceSingleGameMaxMoves as UASymbol]; - gameSingleGameMaxMoves = [thisGameSingleGameMaxMoves]; - } else if (thisSingleGameMaxMoves === singleGameMaxMoves) { - pieceSingleGameMaxMoves.push(thisPieceSingleGameMaxMoves as UASymbol); - gameSingleGameMaxMoves.push(thisGameSingleGameMaxMoves); - } - - const thisGamesNoCastling = - analysis['PieceLevelMoveInfoMetric']['gamesWithNoCastling']; - gamesNoCastling += thisGamesNoCastling; - - const thisQueenKingCastlingCounts = - analysis['PieceLevelMoveInfoMetric']['queenKingCastlingCounts']; - for (const count in thisQueenKingCastlingCounts) { - queenKingCastlingCounts[count] += thisQueenKingCastlingCounts[count]; - } + ({ + singleGameMaxMoves, + pieceSingleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + } = newFunction( + analysis, + totalMovesByPiece, + singleGameMaxMoves, + pieceSingleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + queenKingCastlingCounts + )); // misc move fact metrics const thisEnPassantMovesCount = @@ -283,59 +257,158 @@ async function aggregateResults(filePath: string) { weightedTotalRatingDiff / totalGamesAnalyzedForRatings; // calculating KD Ratios and maxes for final maps - for (const uas of Object.keys(KDMap)) { - const kills = KDMap[uas].kills; - const deaths = KDMap[uas].deaths || 0; - if (deaths !== 0) { - KDRatios[uas] = kills / deaths; - } - } - for (const uas of Object.keys(KDValuesMap)) { - const valueKills = KDValuesMap[uas].valueKills; - const deaths = KDValuesMap[uas].deaths || 0; - if (deaths !== 0) { - KDRatiosValues[uas] = valueKills / deaths; - } - } - for (const uas of Object.keys(KDRatios)) { - if (KDRatios[uas] > maxKDRatio) { - maxKDRatio = KDRatios[uas]; - pieceWithHighestKDRatio = [uas as UASymbol]; - } else if (KDRatios[uas] === maxKDRatio) { - pieceWithHighestKDRatio.push(uas as UASymbol); // tie, add to the array - } - } - for (const uas of Object.keys(KDRatiosValues)) { - if (KDRatiosValues[uas] > maxKDRatioValues) { - maxKDRatioValues = KDRatiosValues[uas]; - pieceWithHighestKDRatioValues = [uas as UASymbol]; - } else if (KDRatiosValues[uas] === maxKDRatio) { - pieceWithHighestKDRatioValues.push(uas as UASymbol); // tie, add to the array - } - } + ({ + maxKDRatio, + pieceWithHighestKDRatio, + maxKDRatioValues, + pieceWithHighestKDRatioValues, + } = kdRatioMetrics( + KDMap, + KDRatios, + KDValuesMap, + KDRatiosValues, + maxKDRatio, + pieceWithHighestKDRatio, + maxKDRatioValues, + pieceWithHighestKDRatioValues + )); // calculating averageNumMovesByPiece (without doing weighted averages) and related maxes - for (const uas in totalMovesByPiece) { - if (!averageNumMovesByPiece[uas]) { - averageNumMovesByPiece[uas] = { - avgNumMoves: totalMovesByPiece[uas].numMoves / totalGamesAnalyzed, - }; - } - } - - for (const uas in averageNumMovesByPiece) { - if (averageNumMovesByPiece[uas].avgNumMoves > highestAverageMoves) { - highestAverageMoves = averageNumMovesByPiece[uas].avgNumMoves; - pieceHighestAverageMoves = [uas as UASymbol]; - } else if ( - averageNumMovesByPiece[uas].avgNumMoves === highestAverageMoves - ) { - pieceHighestAverageMoves.push(uas as UASymbol); - } - } + ({ highestAverageMoves, pieceHighestAverageMoves } = avgNumMoves( + totalMovesByPiece, + averageNumMovesByPiece, + totalGamesAnalyzed, + highestAverageMoves, + pieceHighestAverageMoves + )); // LOGS FOR THE ENTIRE SET // metadata logs + logResults( + weightedAveragePlayerRating, + weightedAverageRatingDiff, + largestRatingDiff, + largestRatingDiffGame, + playerMostGames, + mostGamesPlayedByPlayer, + gameTypeStats, + gameTimeControlStats, + totalGamesAnalyzed, + openings, + bongcloudAppearances, + gameEndings, + KDMap, + KDRatios, + pieceWithHighestKDRatio, + maxKDRatio, + KDValuesMap, + KDRatiosValues, + pieceWithHighestKDRatioValues, + maxKDRatioValues, + KillStreakMap, + maxKillStreak, + maxKillStreakPiece, + maxKillStreakGame, + mateAndAssistMap, + matedCountsMap, + promotedToTotals, + uasPromotingPieces, + maxNumQueens, + movesAndGamesMaxQueens, + pieceMaxAvgDist, + maxAvgDistance, + pieceMinAvgDist, + minAvgDistance, + pieceMaxDistSingleGame, + distPieceMaxDist, + gamePieceMaxDist, + totalCollectiveDistGames, + gameMaxCollectiveDist, + totalDistByPiece, + avgDistByPiece, + gameMostMoves, + gameMostMovesNumMoves, + totalMovesByPiece, + averageNumMovesByPiece, + pieceHighestAverageMoves, + highestAverageMoves, + pieceSingleGameMaxMoves, + singleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + queenKingCastlingCounts, + enPassantMovesCount, + totalNumPiecesKnightHopped, + analysisCounter + ); +} + +console.time('Total Final Analysis Execution Time'); +aggregateResults('src/results.json'); +console.timeEnd('Total Final Analysis Execution Time'); + +function logResults( + weightedAveragePlayerRating: number, + weightedAverageRatingDiff: number, + largestRatingDiff: number, + largestRatingDiffGame: any[], + playerMostGames: any[], + mostGamesPlayedByPlayer: number, + gameTypeStats: {}, + gameTimeControlStats: {}, + totalGamesAnalyzed: number, + openings: {}, + bongcloudAppearances: number, + gameEndings: {}, + KDMap: {}, + KDRatios: {}, + pieceWithHighestKDRatio: any[], + maxKDRatio: number, + KDValuesMap: {}, + KDRatiosValues: {}, + pieceWithHighestKDRatioValues: any[], + maxKDRatioValues: number, + KillStreakMap: {}, + maxKillStreak: number, + maxKillStreakPiece: any[], + maxKillStreakGame: any[], + mateAndAssistMap: {}, + matedCountsMap: { k: number; K: number }, + promotedToTotals: { q: number; r: number; b: number; n: number }, + uasPromotingPieces: {}, + maxNumQueens: number, + movesAndGamesMaxQueens: any[], + pieceMaxAvgDist: any[], + maxAvgDistance: number, + pieceMinAvgDist: any[], + minAvgDistance: number, + pieceMaxDistSingleGame: any[], + distPieceMaxDist: number, + gamePieceMaxDist: any[], + totalCollectiveDistGames: number, + gameMaxCollectiveDist: { distance: number; games: any[] }, + totalDistByPiece: {}, + avgDistByPiece: {}, + gameMostMoves: any[], + gameMostMovesNumMoves: number, + totalMovesByPiece: {}, + averageNumMovesByPiece: {}, + pieceHighestAverageMoves: any[], + highestAverageMoves: number, + pieceSingleGameMaxMoves: any[], + singleGameMaxMoves: number, + gameSingleGameMaxMoves: any[], + gamesNoCastling: number, + queenKingCastlingCounts: { + blackKing: number; + blackQueen: number; + whiteKing: number; + whiteQueen: number; + }, + enPassantMovesCount: number, + totalNumPiecesKnightHopped: number, + analysisCounter: number +) { console.log('GAME SET STATS (METADATA) ----------------------------'); console.log(`Average Player Rating: ${weightedAveragePlayerRating}`); console.log(`Average Rating Difference: ${weightedAverageRatingDiff}`); @@ -512,9 +585,155 @@ async function aggregateResults(filePath: string) { console.log(`Number of separate analyses: ${analysisCounter}`); } -console.time('Total Final Analysis Execution Time'); -aggregateResults('src/results.json'); -console.timeEnd('Total Final Analysis Execution Time'); +function avgNumMoves( + totalMovesByPiece: {}, + averageNumMovesByPiece: {}, + totalGamesAnalyzed: number, + highestAverageMoves: number, + pieceHighestAverageMoves: any[] +) { + for (const uas in totalMovesByPiece) { + if (!averageNumMovesByPiece[uas]) { + averageNumMovesByPiece[uas] = { + avgNumMoves: totalMovesByPiece[uas].numMoves / totalGamesAnalyzed, + }; + } + } + + for (const uas in averageNumMovesByPiece) { + if (averageNumMovesByPiece[uas].avgNumMoves > highestAverageMoves) { + highestAverageMoves = averageNumMovesByPiece[uas].avgNumMoves; + pieceHighestAverageMoves = [uas as UASymbol]; + } else if ( + averageNumMovesByPiece[uas].avgNumMoves === highestAverageMoves + ) { + pieceHighestAverageMoves.push(uas as UASymbol); + } + } + return { highestAverageMoves, pieceHighestAverageMoves }; +} + +function kdRatioMetrics( + KDMap: {}, + KDRatios: {}, + KDValuesMap: {}, + KDRatiosValues: {}, + maxKDRatio: number, + pieceWithHighestKDRatio: any[], + maxKDRatioValues: number, + pieceWithHighestKDRatioValues: any[] +) { + for (const uas of Object.keys(KDMap)) { + const kills = KDMap[uas].kills; + const deaths = KDMap[uas].deaths || 0; + if (deaths !== 0) { + KDRatios[uas] = kills / deaths; + } + } + for (const uas of Object.keys(KDValuesMap)) { + const valueKills = KDValuesMap[uas].valueKills; + const deaths = KDValuesMap[uas].deaths || 0; + if (deaths !== 0) { + KDRatiosValues[uas] = valueKills / deaths; + } + } + for (const uas of Object.keys(KDRatios)) { + if (KDRatios[uas] > maxKDRatio) { + maxKDRatio = KDRatios[uas]; + pieceWithHighestKDRatio = [uas as UASymbol]; + } else if (KDRatios[uas] === maxKDRatio) { + pieceWithHighestKDRatio.push(uas as UASymbol); // tie, add to the array + } + } + for (const uas of Object.keys(KDRatiosValues)) { + if (KDRatiosValues[uas] > maxKDRatioValues) { + maxKDRatioValues = KDRatiosValues[uas]; + pieceWithHighestKDRatioValues = [uas as UASymbol]; + } else if (KDRatiosValues[uas] === maxKDRatio) { + pieceWithHighestKDRatioValues.push(uas as UASymbol); // tie, add to the array + } + } + return { + maxKDRatio, + pieceWithHighestKDRatio, + maxKDRatioValues, + pieceWithHighestKDRatioValues, + }; +} + +function newFunction( + analysis: unknown, + totalMovesByPiece: {}, + singleGameMaxMoves: number, + pieceSingleGameMaxMoves: any[], + gameSingleGameMaxMoves: any[], + gamesNoCastling: number, + queenKingCastlingCounts: { + blackKing: number; + blackQueen: number; + whiteKing: number; + whiteQueen: number; + } +) { + const thisTotalMovesByPiece = + analysis['PieceLevelMoveInfoMetric']['totalMovesByPiece']; + for (const uas in thisTotalMovesByPiece) { + if (!totalMovesByPiece[uas]) { + totalMovesByPiece[uas] = { + numMoves: thisTotalMovesByPiece[uas].numMoves, + }; + } + totalMovesByPiece[uas].numMoves += thisTotalMovesByPiece[uas].numMoves; + } + + const thisSingleGameMaxMoves = + analysis['PieceLevelMoveInfoMetric']['uasSingleGameMaxMoves']; + const thisPieceSingleGameMaxMoves = + analysis['PieceLevelMoveInfoMetric']['uasWithMostMovesSingleGame']; + const thisGameSingleGameMaxMoves = + analysis['PieceLevelMoveInfoMetric']['gamesWithUasMostMoves']; + if (thisSingleGameMaxMoves > singleGameMaxMoves) { + singleGameMaxMoves = thisSingleGameMaxMoves; + pieceSingleGameMaxMoves = [thisPieceSingleGameMaxMoves as UASymbol]; + gameSingleGameMaxMoves = [thisGameSingleGameMaxMoves]; + } else if (thisSingleGameMaxMoves === singleGameMaxMoves) { + pieceSingleGameMaxMoves.push(thisPieceSingleGameMaxMoves as UASymbol); + gameSingleGameMaxMoves.push(thisGameSingleGameMaxMoves); + } + + const thisGamesNoCastling = + analysis['PieceLevelMoveInfoMetric']['gamesWithNoCastling']; + gamesNoCastling += thisGamesNoCastling; + + const thisQueenKingCastlingCounts = + analysis['PieceLevelMoveInfoMetric']['queenKingCastlingCounts']; + for (const count in thisQueenKingCastlingCounts) { + queenKingCastlingCounts[count] += thisQueenKingCastlingCounts[count]; + } + return { + singleGameMaxMoves, + pieceSingleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + }; +} + +function aggregateRatingDiff( + analysis: unknown, + largestRatingDiff: number, + largestRatingDiffGame: any[] +) { + const thisLargestRatingDiff = analysis['MetadataMetric']['largestRatingDiff']; + const thisLargestRatingDiffGame = + analysis['MetadataMetric']['largestRatingDiffGame']; + if (thisLargestRatingDiff > largestRatingDiff) { + largestRatingDiff = thisLargestRatingDiff; + largestRatingDiffGame = thisLargestRatingDiffGame; + } else if (thisLargestRatingDiff === largestRatingDiff) { + largestRatingDiffGame.push(thisLargestRatingDiffGame); + } + return { largestRatingDiff, largestRatingDiffGame }; +} function aggregateDistanceMetrics( analysis: unknown, diff --git a/src/zst_decompressor.ts b/src/zst_decompressor.ts index b138d6b..c422d77 100644 --- a/src/zst_decompressor.ts +++ b/src/zst_decompressor.ts @@ -5,18 +5,15 @@ const fs = require('fs'); const zstd = require('node-zstandard'); const { spawn } = require('child_process'); -// List of all the database files you want to analyze (these need to be downloaded and in data folder) -const files = ['lichess_db_standard_rated_2018-05.pgn.zst' /*...*/]; - // 30 games = 10*1024 bytes, 1 game = 350 bytes, 1000 games = 330KB, 100K games = 33MB // 10MB yields around 30k games, 5GB = around 15 million games // const SIZE_LIMIT = 30 * 1024 * 1024; // 30MB -const SIZE_LIMIT = 0.5 * 1024 * 1024; // 0.5MB, for testing +const SIZE_LIMIT = 30 * 1024 * 1024; // 0.5MB, for testing // set the total size limit of the combined decompressed files (this is how much space you need to have available on your PC prior to running node src/streaming_partial_decompresser.js) const decompressedSizeLimit = 500 * 1024 * 1024 * 1024; // 500 GB represented in bytes -const getFileSize = (filePath) => { +const getFileSize = (filePath: string) => { if (!fs.existsSync(filePath)) { return 0; } @@ -29,14 +26,14 @@ const getFileSize = (filePath) => { * @param {string} filePath - The path of the file to run the analysis on. * @return {Promise} A promise that resolves when the analysis is complete. */ -async function runAnalysis(filePath) { +async function runAnalysis(filePath: string) { return new Promise((resolve, reject) => { // Run the analysis script console.log(`Running analysis script on ${filePath}...`); const child = spawn('ts-node', [ // '/Users/bennyrubanov/Coding_Projects/chessanalysis/src/index_with_decompressor.ts', - `${__dirname}/../run_metrics_on_input.ts`, + `${__dirname}/../../run_metrics_on_input.ts`, filePath, ]); @@ -85,7 +82,7 @@ const decompressAndAnalyze = async (file, start = 0) => { const filesProduced = new Set(); // const base_path = `/Users/bennyrubanov/Coding_Projects/chessanalysis/data/${file.replace( - const base_path = `${__dirname}/../data/${file.replace('.zst', '')}`; + const base_path = `${__dirname}/../../data/${file.replace('.zst', '')}`; // Create a new file path const newFilePath = `${base_path}_${randomUUID()}`; @@ -111,7 +108,7 @@ const decompressAndAnalyze = async (file, start = 0) => { // https://www.npmjs.com/package/node-zstandard#decompressionstreamfromfile-inputfile-callback zstd.decompressionStreamFromFile( - `${__dirname}/../data/${file}`, + `${__dirname}/../../data/${file}`, (err, result) => { if (err) return reject(err); @@ -212,7 +209,7 @@ const decompressAndAnalyze = async (file, start = 0) => { }; // Function to process all files -const processFiles = async () => { +const processFiles = async (files: string[]) => { console.log(`Initiating decompression and analysis of ${files}...`); console.time('Final Total Compressed File Analysis Execution Time'); for (const file of files) { @@ -221,9 +218,6 @@ const processFiles = async () => { console.timeEnd('Final Total Compressed File Analysis Execution Time'); }; -// Start the process -processFiles(); - const formatDuration = (duration) => { const hours = Math.floor(duration / 3600000); const minutes = Math.floor((duration % 3600000) / 60000); @@ -234,3 +228,10 @@ const formatDuration = (duration) => { }; module.exports = processFiles; + +// run if main +if (require.main === module) { + // List of all the database files you want to analyze (these need to be downloaded and in data folder) + const files = ['lichess_db_standard_rated_2013-02.pgn.zst' /*...*/]; + processFiles(files); +}