diff --git a/package-lock.json b/package-lock.json index 2d1d717..bdb7eed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,14 +13,17 @@ "chess.js": "^1.0.0-beta.6", "d3": "^7.8.5", "node-zstandard": "^1.2.4", - "proper-lockfile": "^4.1.2" + "proper-lockfile": "^4.1.2", + "ts-node": "10.9.2" }, "devDependencies": { "@babel/preset-typescript": "^7.23.0", + "@types/async": "3.2.24", "@types/d3": "^7.4.1", "@types/jest": "^29.5.5", "@types/lodash": "^4.14.199", "@types/node": "18.11.18", + "@types/proper-lockfile": "4.1.4", "jest": "^29.7.0", "ts-jest": "^29.1.1" } @@ -808,6 +811,26 @@ "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", "dev": true }, + "node_modules/@cspotcode/source-map-support": { + "version": "0.8.1", + "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", + "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", + "dependencies": { + "@jridgewell/trace-mapping": "0.3.9" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", + "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.0.3", + "@jridgewell/sourcemap-codec": "^1.4.10" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -1129,7 +1152,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.1.tgz", "integrity": "sha512-dSYZh7HhCDtCKm4QakX0xFpsRDqjjtZf/kjI/v3T3Nwt5r8/qz/M19F9ySyOqU94SXBmeG9ttTul+YnR4LOxFA==", - "dev": true, "engines": { "node": ">=6.0.0" } @@ -1146,8 +1168,7 @@ "node_modules/@jridgewell/sourcemap-codec": { "version": "1.4.15", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "dev": true + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.19", @@ -1183,6 +1204,32 @@ "@sinonjs/commons": "^3.0.0" } }, + "node_modules/@tsconfig/node10": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.9.tgz", + "integrity": "sha512-jNsYVVxU8v5g43Erja32laIDHXeoNvFEpX33OK4d6hljo3jDhCBDhx5dhCCTMWUojscpAagGiRkBKxpdl9fxqA==" + }, + "node_modules/@tsconfig/node12": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", + "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==" + }, + "node_modules/@tsconfig/node14": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", + "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==" + }, + "node_modules/@tsconfig/node16": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", + "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==" + }, + "node_modules/@types/async": { + "version": "3.2.24", + "resolved": "https://registry.npmjs.org/@types/async/-/async-3.2.24.tgz", + "integrity": "sha512-8iHVLHsCCOBKjCF2KwFe0p9Z3rfM9mL+sSP8btyR5vTjJRAqpBYD28/ZLgXPf0pjG1VxOvtCV/BgXkQbpSe8Hw==", + "dev": true + }, "node_modules/@types/babel__core": { "version": "7.20.2", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.2.tgz", @@ -1535,7 +1582,21 @@ "node_modules/@types/node": { "version": "18.11.18", "resolved": "https://registry.npmjs.org/@types/node/-/node-18.11.18.tgz", - "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==", + "integrity": "sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==" + }, + "node_modules/@types/proper-lockfile": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/@types/proper-lockfile/-/proper-lockfile-4.1.4.tgz", + "integrity": "sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==", + "dev": true, + "dependencies": { + "@types/retry": "*" + } + }, + "node_modules/@types/retry": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.5.tgz", + "integrity": "sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==", "dev": true }, "node_modules/@types/stack-utils": { @@ -1559,6 +1620,25 @@ "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==", "dev": true }, + "node_modules/acorn": { + "version": "8.11.3", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", + "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-walk": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.2.tgz", + "integrity": "sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/ansi-escapes": { "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", @@ -1611,6 +1691,11 @@ "node": ">= 8" } }, + "node_modules/arg": { + "version": "4.1.3", + "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", + "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==" + }, "node_modules/argparse": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", @@ -2005,6 +2090,11 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/create-require": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", + "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2446,6 +2536,14 @@ "node": ">=8" } }, + "node_modules/diff": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", + "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", + "engines": { + "node": ">=0.3.1" + } + }, "node_modules/diff-sequences": { "version": "29.6.3", "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", @@ -3706,8 +3804,7 @@ "node_modules/make-error": { "version": "1.3.6", "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true + "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==" }, "node_modules/makeerror": { "version": "1.0.12", @@ -4420,6 +4517,48 @@ "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", "dev": true }, + "node_modules/ts-node": { + "version": "10.9.2", + "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", + "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", + "dependencies": { + "@cspotcode/source-map-support": "^0.8.0", + "@tsconfig/node10": "^1.0.7", + "@tsconfig/node12": "^1.0.7", + "@tsconfig/node14": "^1.0.0", + "@tsconfig/node16": "^1.0.2", + "acorn": "^8.4.1", + "acorn-walk": "^8.1.1", + "arg": "^4.1.0", + "create-require": "^1.1.0", + "diff": "^4.0.1", + "make-error": "^1.1.1", + "v8-compile-cache-lib": "^3.0.1", + "yn": "3.1.1" + }, + "bin": { + "ts-node": "dist/bin.js", + "ts-node-cwd": "dist/bin-cwd.js", + "ts-node-esm": "dist/bin-esm.js", + "ts-node-script": "dist/bin-script.js", + "ts-node-transpile-only": "dist/bin-transpile.js", + "ts-script": "dist/bin-script-deprecated.js" + }, + "peerDependencies": { + "@swc/core": ">=1.2.50", + "@swc/wasm": ">=1.2.50", + "@types/node": "*", + "typescript": ">=2.7" + }, + "peerDependenciesMeta": { + "@swc/core": { + "optional": true + }, + "@swc/wasm": { + "optional": true + } + } + }, "node_modules/tslib": { "version": "2.6.2", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", @@ -4450,7 +4589,6 @@ "version": "5.2.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", - "dev": true, "peer": true, "bin": { "tsc": "bin/tsc", @@ -4490,6 +4628,11 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/v8-compile-cache-lib": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", + "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==" + }, "node_modules/v8-to-istanbul": { "version": "9.1.0", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.0.tgz", @@ -4612,6 +4755,14 @@ "node": ">=12" } }, + "node_modules/yn": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", + "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", + "engines": { + "node": ">=6" + } + }, "node_modules/yocto-queue": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", diff --git a/package.json b/package.json index 999c87d..045da9f 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,8 @@ "chess.js": "^1.0.0-beta.6", "d3": "^7.8.5", "node-zstandard": "^1.2.4", - "proper-lockfile": "^4.1.2" + "proper-lockfile": "^4.1.2", + "ts-node": "10.9.2" }, "devDependencies": { "@babel/preset-typescript": "^7.23.0", @@ -16,6 +17,8 @@ "@types/jest": "^29.5.5", "@types/lodash": "^4.14.199", "@types/node": "18.11.18", + "@types/proper-lockfile": "4.1.4", + "@types/async": "3.2.24", "jest": "^29.7.0", "ts-jest": "^29.1.1" }, @@ -23,6 +26,7 @@ "build": "tsc --build", "start": "tsc --build && node dist/src/index.js", "test": "tsc --build && jest ./dist", - "scratch": "tsc --build && node dist/src/scratch.js" + "scratch": "tsc --build && node dist/src/scratch.js", + "run-zst-decompressor": "tsc --build && node dist/src/zst_decompressor.js" } } diff --git a/src/aggregate_analysis.ts b/src/aggregate_analysis.ts new file mode 100644 index 0000000..580c774 --- /dev/null +++ b/src/aggregate_analysis.ts @@ -0,0 +1,1054 @@ +import { readFileSync } from 'fs'; +import { UASymbol } from '../cjsmin/src/chess'; + +/** + * + * @param results.json + * @returns final analysis results of all files created and deleted in zst_decompressor.ts + */ +async function aggregateResults(filePath: string) { + const data = JSON.parse(readFileSync(filePath, 'utf-8')); + + // instantiate final variables + let totalGamesAnalyzed = 0; + let analysisCounter = 0; + + // metadata metrics + let largestRatingDiff = 0; + let largestRatingDiffGame = []; + let mostGamesPlayedByPlayer = 0; + let playerMostGames = []; + let gameTypeStats = {}; + let gameTimeControlStats = {}; + let openings = {}; + let bongcloudAppearances = 0; + let gameEndings = {}; + let totalGamesAnalyzedForRatings = 0; // accounting for some missing rating data + + // capture metrics + let KDMap = {}; + let KDValuesMap = {}; + let KDRatios = {}; + let KDRatiosValues = {}; + let maxKDRatio = 0; + let maxKDRatioValues = 0; + let pieceWithHighestKDRatio = []; + let pieceWithHighestKDRatioValues = []; + let KillStreakMap = {}; + let maxKillStreak = 0; + let maxKillStreakPiece = []; + let maxKillStreakGame = []; + + // mates and assists metrics + let mateAndAssistMap = {}; + let matedCountsMap = { + k: 0, + K: 0, + }; + + // promotions metrics + let promotedToTotals = { + q: 0, + r: 0, + b: 0, + n: 0, + }; + let uasPromotingPieces = {}; + let maxNumQueens = 0; + let movesAndGamesMaxQueens = []; + + // distance metrics + let pieceMaxAvgDist = []; + let maxAvgDistance = 0; + let pieceMinAvgDist = []; + let minAvgDistance = Infinity; + let pieceMaxDistSingleGame = []; + let gamePieceMaxDist = []; + let distPieceMaxDist = 0; + let totalCollectiveDistGames = 0; + let gameMaxCollectiveDist = { + distance: 0, + games: [], + }; + let totalDistByPiece = {}; + let avgDistByPiece = {}; + + // moves metrics + let gameMostMoves = []; + let gameMostMovesNumMoves = 0; + let totalMovesByPiece = {}; + let averageNumMovesByPiece = {}; + let pieceHighestAverageMoves = []; + let highestAverageMoves = 0; + let singleGameMaxMoves = 0; + let pieceSingleGameMaxMoves = []; + let gameSingleGameMaxMoves = []; + let gamesNoCastling = 0; + let queenKingCastlingCounts = { + blackKing: 0, + blackQueen: 0, + whiteKing: 0, + whiteQueen: 0, + }; + let enPassantMovesCount = 0; + let totalNumPiecesKnightHopped = 0; + + // helper variables + let weightedTotalPlayerRating = 0; + let weightedTotalRatingDiff = 0; + + // ANALYSIS-BY-ANALYSIS CALCULATIONS + for (const analysis of Object.values(data)) { + analysisCounter++; + + const thisAnalysisGamesAnalyzed = analysis['Number of games analyzed']; + + // METADATA METRICS + // ratings weighted average calculations + ({ + weightedTotalPlayerRating, + weightedTotalRatingDiff, + totalGamesAnalyzedForRatings, + } = aggregateMetadata( + analysis, + weightedTotalPlayerRating, + weightedTotalRatingDiff, + totalGamesAnalyzedForRatings + )); + + // ratings largest diff + ({ largestRatingDiff, largestRatingDiffGame } = aggregateRatingDiff( + analysis, + largestRatingDiff, + largestRatingDiffGame + )); + + // games played + // currently this stat is inaccurately tracked across analyses + // the player with the most games might have their games split across analyses + // the fix is to return the playerGameStats from misc.ts and then parse that in the final aggregate analysis but that dataset could be very large + const thisMostGamesPlayed = analysis['MetadataMetric']['mostGamesPlayed']; + const thisPlayerMostGames = analysis['MetadataMetric']['playerMostGames']; + if (thisMostGamesPlayed > mostGamesPlayedByPlayer) { + mostGamesPlayedByPlayer = thisMostGamesPlayed; + playerMostGames = thisPlayerMostGames; + } else if (thisMostGamesPlayed === mostGamesPlayedByPlayer) { + playerMostGames.push(thisPlayerMostGames); + } + + // aggregate gameTypeStats + aggregateGameTypeStats(analysis, gameTypeStats); + + // aggregate gameTimeControlStats + aggregateGameTimeControlStats(analysis, gameTimeControlStats); + + // aggregate openings stats + bongcloudAppearances = aggregateOpeningStats( + analysis, + openings, + bongcloudAppearances + ); + + // aggregate game endings stats + aggregateEndingStats(analysis, gameEndings); + + // KD AND CAPTURE METRICS + // KD Ratios + // Recalculating KD Ratios across all the analyses (alternatively could do weighted averages) + aggregateKDRatio(analysis, KDMap, KDValuesMap); + + // kill streaks + ({ maxKillStreak, maxKillStreakPiece, maxKillStreakGame } = + aggregateKillStreaks( + analysis, + KillStreakMap, + maxKillStreak, + maxKillStreakPiece, + maxKillStreakGame + )); + + // mates and assists + aggregateMatesAndAssists(analysis, mateAndAssistMap, matedCountsMap); + + // promotions metrics + const { thisMaxNumQueens, thisMovesAndGamesMaxQueens } = + aggregatePromotions(analysis, promotedToTotals, uasPromotingPieces); + + // find maxes + if (thisMaxNumQueens > maxNumQueens) { + maxNumQueens = thisMaxNumQueens; + movesAndGamesMaxQueens = thisMovesAndGamesMaxQueens; + } else if (thisMaxNumQueens > maxNumQueens) { + movesAndGamesMaxQueens.push(thisMovesAndGamesMaxQueens); + } + + // distance metrics + ({ + maxAvgDistance, + pieceMaxAvgDist, + minAvgDistance, + pieceMinAvgDist, + distPieceMaxDist, + pieceMaxDistSingleGame, + gamePieceMaxDist, + totalCollectiveDistGames, + gameMaxCollectiveDist, + } = aggregateDistanceMetrics( + analysis, + maxAvgDistance, + pieceMaxAvgDist, + minAvgDistance, + pieceMinAvgDist, + distPieceMaxDist, + pieceMaxDistSingleGame, + gamePieceMaxDist, + totalCollectiveDistGames, + gameMaxCollectiveDist, + totalDistByPiece, + avgDistByPiece + )); + + // moves metrics + const thisGameMostMoves = + analysis['GameWithMostMovesMetric']['gameWithMostMoves']; + const thisGameMostMovesNumMoves = + analysis['GameWithMostMovesMetric']['gameWithMostMovesNumMoves']; + if (thisGameMostMovesNumMoves > gameMostMovesNumMoves) { + gameMostMovesNumMoves = thisGameMostMovesNumMoves; + gameMostMoves = [thisGameMostMoves]; + } else if (thisGameMostMovesNumMoves === gameMostMovesNumMoves) { + gameMostMoves.push(thisGameMostMoves); + } + + // piece level moves metrics + ({ + singleGameMaxMoves, + pieceSingleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + } = newFunction( + analysis, + totalMovesByPiece, + singleGameMaxMoves, + pieceSingleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + queenKingCastlingCounts + )); + + // misc move fact metrics + const thisEnPassantMovesCount = + analysis['MiscMoveFactMetric']['enPassantMovesCount']; + enPassantMovesCount += thisEnPassantMovesCount; + + const thisTotalNumPiecesKnightHopped = + analysis['MiscMoveFactMetric']['totalNumPiecesKnightHopped']; + totalNumPiecesKnightHopped += thisTotalNumPiecesKnightHopped; + + // final increments + totalGamesAnalyzed += thisAnalysisGamesAnalyzed; + } + + // AGGREGATE CALCULATIONS + // ratings weighted average calculations + const weightedAveragePlayerRating = + weightedTotalPlayerRating / totalGamesAnalyzedForRatings; + const weightedAverageRatingDiff = + weightedTotalRatingDiff / totalGamesAnalyzedForRatings; + + // calculating KD Ratios and maxes for final maps + ({ + maxKDRatio, + pieceWithHighestKDRatio, + maxKDRatioValues, + pieceWithHighestKDRatioValues, + } = kdRatioMetrics( + KDMap, + KDRatios, + KDValuesMap, + KDRatiosValues, + maxKDRatio, + pieceWithHighestKDRatio, + maxKDRatioValues, + pieceWithHighestKDRatioValues + )); + + // calculating averageNumMovesByPiece (without doing weighted averages) and related maxes + ({ highestAverageMoves, pieceHighestAverageMoves } = avgNumMoves( + totalMovesByPiece, + averageNumMovesByPiece, + totalGamesAnalyzed, + highestAverageMoves, + pieceHighestAverageMoves + )); + + // LOGS FOR THE ENTIRE SET + // metadata logs + logResults( + weightedAveragePlayerRating, + weightedAverageRatingDiff, + largestRatingDiff, + largestRatingDiffGame, + playerMostGames, + mostGamesPlayedByPlayer, + gameTypeStats, + gameTimeControlStats, + totalGamesAnalyzed, + openings, + bongcloudAppearances, + gameEndings, + KDMap, + KDRatios, + pieceWithHighestKDRatio, + maxKDRatio, + KDValuesMap, + KDRatiosValues, + pieceWithHighestKDRatioValues, + maxKDRatioValues, + KillStreakMap, + maxKillStreak, + maxKillStreakPiece, + maxKillStreakGame, + mateAndAssistMap, + matedCountsMap, + promotedToTotals, + uasPromotingPieces, + maxNumQueens, + movesAndGamesMaxQueens, + pieceMaxAvgDist, + maxAvgDistance, + pieceMinAvgDist, + minAvgDistance, + pieceMaxDistSingleGame, + distPieceMaxDist, + gamePieceMaxDist, + totalCollectiveDistGames, + gameMaxCollectiveDist, + totalDistByPiece, + avgDistByPiece, + gameMostMoves, + gameMostMovesNumMoves, + totalMovesByPiece, + averageNumMovesByPiece, + pieceHighestAverageMoves, + highestAverageMoves, + pieceSingleGameMaxMoves, + singleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + queenKingCastlingCounts, + enPassantMovesCount, + totalNumPiecesKnightHopped, + analysisCounter + ); +} + +function logResults( + weightedAveragePlayerRating: number, + weightedAverageRatingDiff: number, + largestRatingDiff: number, + largestRatingDiffGame: any[], + playerMostGames: any[], + mostGamesPlayedByPlayer: number, + gameTypeStats: {}, + gameTimeControlStats: {}, + totalGamesAnalyzed: number, + openings: {}, + bongcloudAppearances: number, + gameEndings: {}, + KDMap: {}, + KDRatios: {}, + pieceWithHighestKDRatio: any[], + maxKDRatio: number, + KDValuesMap: {}, + KDRatiosValues: {}, + pieceWithHighestKDRatioValues: any[], + maxKDRatioValues: number, + KillStreakMap: {}, + maxKillStreak: number, + maxKillStreakPiece: any[], + maxKillStreakGame: any[], + mateAndAssistMap: {}, + matedCountsMap: { k: number; K: number }, + promotedToTotals: { q: number; r: number; b: number; n: number }, + uasPromotingPieces: {}, + maxNumQueens: number, + movesAndGamesMaxQueens: any[], + pieceMaxAvgDist: any[], + maxAvgDistance: number, + pieceMinAvgDist: any[], + minAvgDistance: number, + pieceMaxDistSingleGame: any[], + distPieceMaxDist: number, + gamePieceMaxDist: any[], + totalCollectiveDistGames: number, + gameMaxCollectiveDist: { distance: number; games: any[] }, + totalDistByPiece: {}, + avgDistByPiece: {}, + gameMostMoves: any[], + gameMostMovesNumMoves: number, + totalMovesByPiece: {}, + averageNumMovesByPiece: {}, + pieceHighestAverageMoves: any[], + highestAverageMoves: number, + pieceSingleGameMaxMoves: any[], + singleGameMaxMoves: number, + gameSingleGameMaxMoves: any[], + gamesNoCastling: number, + queenKingCastlingCounts: { + blackKing: number; + blackQueen: number; + whiteKing: number; + whiteQueen: number; + }, + enPassantMovesCount: number, + totalNumPiecesKnightHopped: number, + analysisCounter: number +) { + console.log('GAME SET STATS (METADATA) ----------------------------'); + console.log(`Average Player Rating: ${weightedAveragePlayerRating}`); + console.log(`Average Rating Difference: ${weightedAverageRatingDiff}`); + console.log(`Largest Rating Difference: ${largestRatingDiff}`); + console.log(`Largest Rating Difference Game(s): ${largestRatingDiffGame}`); + console.log( + `Player(s) with the most games played (CURRENTLY INACCURATELY TRACKED): ${playerMostGames}` + ); + console.log( + `Number of games played (CURRENTLY INACCURATELY TRACKED): ${mostGamesPlayedByPlayer}` + ); + console.log('\n'); + console.log(`Game Type Stats: `), console.table(gameTypeStats); + console.log( + `Time Control Stats: (filtered by appearing in at least 1% of games):` + ); + + const sortedGameTimeControlStats = Object.entries(gameTimeControlStats).sort( + ([, valueA], [, valueB]) => Number(valueB) - Number(valueA) + ); + + const filteredGameTimeControlStats = Object.fromEntries( + sortedGameTimeControlStats.filter( + ([_, value]) => (value as number) / totalGamesAnalyzed > 0.01 + ) + ); + + console.table(filteredGameTimeControlStats); + + console.log( + 'Openings stats (filtered by appearing in at least 1% of games):' + ); + + const sortedOpenings = Object.entries(openings).sort( + ( + [, dataA]: [string, { whiteToBlackWinRatio: number | null }], + [, dataB]: [string, { whiteToBlackWinRatio: number | null }] + ) => (dataB.whiteToBlackWinRatio || 0) - (dataA.whiteToBlackWinRatio || 0) + ); + const filteredOpenings = Object.fromEntries( + sortedOpenings.filter( + ([_, data]: [ + string, + { + appearances: number; + blackWins: number; + whiteWins: number; + ties: number; + whiteToBlackWinRatio: number | null; + } + ]) => data.appearances / totalGamesAnalyzed > 0.01 + ) + ); + + console.table(filteredOpenings); + + console.log(`Number of bongcloud appearances: ${bongcloudAppearances}`); + console.log(`Game Endings: `), console.table(gameEndings); + console.log('\n'); + + // captures logs + console.log('CAPTURES STATS: ----------------------------'); + console.log('Kills, deaths, and revenge kills for each unambiguous piece:'), + console.table(KDMap); + console.log( + 'Kill Death Ratios for each unambiguous piece: ' + + JSON.stringify(KDRatios, null, 2) + ); + console.log( + `Piece with the highest KD ratio was ${pieceWithHighestKDRatio} with a ratio of ${maxKDRatio}` + ); + + console.log('\n'); + console.log( + 'Piece values for kills: Pawn 1 point, Knight 3 points, Bishop 3 points, Rook 5 points, Queen 9 points, King 4 points. ' + ); + console.log('Value kills and deaths for each unambiguous piece:'), + console.table(KDValuesMap); + console.log( + 'Kill Death Ratios for each unambiguous piece: ' + + JSON.stringify(KDRatiosValues, null, 2) + ); + console.log( + `Piece with the highest KD ratio (taking into account piece values) was ${pieceWithHighestKDRatioValues} with a ratio of ${maxKDRatioValues}` + ); + + console.log('\n'); + console.log('Max Kill Streaks achieved for each piece: '); + console.table(KillStreakMap); + console.log( + `Max Kill Streak achieved by any piece (the number of captures without any other piece on its team capturing. doesn't have to be consecutive move captures): ${maxKillStreak} by the piece(s) ${maxKillStreakPiece}. This was done in the game(s): ` + ); + console.log(maxKillStreakGame.join('\n')); + + // mates and assists logs + console.log('\n'); + console.log('MATES AND ASSISTS STATS: ----------------------------'); + console.log('Mates, assists, and hockey assists for each piece: '); + console.table(mateAndAssistMap); + console.log( + 'Note: any "mates" attributed to kings are a result of a king moving to reveal a discovered mate.' + ); + console.log('Number of times each king was mated: '); + console.table(matedCountsMap); + + // promotions logs + console.log('\n'); + console.log('PROMOTIONS STATS: ----------------------------'); + console.log('Pieces promoted to most often: '); + console.table(promotedToTotals); + console.log('The pieces each unambiguous piece promotes to most often: '); + console.table(uasPromotingPieces); + console.log( + `The maximum number of queens to appear in a given move in a game: ${maxNumQueens}` + ); + console.log(`The games(s) and first move(s) in that game in which that number of queens appeared: + ${movesAndGamesMaxQueens + .map((move) => JSON.stringify(move, null, 2)) + .join(', ')}`); + + // distance logs + console.log('\n'); + console.log('DISTANCE STATS: ----------------------------'); + console.log( + `Piece(s) with highest average distance: ${pieceMaxAvgDist}. That/those piece(s) average distance: ${maxAvgDistance}` + ); + console.log( + `Piece(s) with lowest average distance: ${pieceMinAvgDist}. That/those piece(s) average distance: ${minAvgDistance}` + ); + console.log( + `Piece that covered the most ground in a single game: ${pieceMaxDistSingleGame}. Distance covered: ${distPieceMaxDist}. Game in which that distance was covered by that piece: ${gamePieceMaxDist}.` + ); + console.log( + `Total collective distance of all pieces in games analyzed: ${totalCollectiveDistGames}` + ); + console.log( + `Game(s) with the furthest collective distance moved: ${gameMaxCollectiveDist.games}` + ); + console.log(`Distance moved: ${gameMaxCollectiveDist.distance}`); + console.log(`Total distance moved by piece:`); + console.table(totalDistByPiece); + console.log(`Average distance moved by piece:`); + console.table(avgDistByPiece); + + // + console.log('\n'); + console.log('MOVES STATS: ----------------------------'); + console.log( + `Game(s) with most moves made (1 move = one white or one black move): ${gameMostMoves}` + ); + console.log(`Number of moves made: ${gameMostMovesNumMoves}`); + console.log('Total number of moves made by each piece: '); + console.table(totalMovesByPiece); + console.log('Average number of moves made by each piece: '); + console.table(averageNumMovesByPiece); + console.log( + `Piece(s) with the highest average number of moves: ${pieceHighestAverageMoves}. The average number of moves that/those pieces made per game: ${highestAverageMoves}` + ); + console.log( + `The piece with the most moves played in a single game: ${pieceSingleGameMaxMoves}. The number of moves played in that game: ${singleGameMaxMoves}. The game it played that number of moves in: ${gameSingleGameMaxMoves}` + ); + console.log(`The number of games with no castling: ${gamesNoCastling}`); + console.log('The number of times each kind of castling happened: '); + console.table(queenKingCastlingCounts); + console.log(`The number of En passants that occured: ${enPassantMovesCount}`); + console.log( + `The number of pieces that were hopped over by a knight: ${totalNumPiecesKnightHopped}` + ); + + // final analysis logs + console.log('\n'); + console.log('ANALYSIS STATS: ----------------------------'); + console.log(`Total games analyzed: ${totalGamesAnalyzed}`); + console.log(`Number of separate analyses: ${analysisCounter}`); +} + +function avgNumMoves( + totalMovesByPiece: {}, + averageNumMovesByPiece: {}, + totalGamesAnalyzed: number, + highestAverageMoves: number, + pieceHighestAverageMoves: any[] +) { + for (const uas in totalMovesByPiece) { + if (!averageNumMovesByPiece[uas]) { + averageNumMovesByPiece[uas] = { + avgNumMoves: totalMovesByPiece[uas].numMoves / totalGamesAnalyzed, + }; + } + } + + for (const uas in averageNumMovesByPiece) { + if (averageNumMovesByPiece[uas].avgNumMoves > highestAverageMoves) { + highestAverageMoves = averageNumMovesByPiece[uas].avgNumMoves; + pieceHighestAverageMoves = [uas as UASymbol]; + } else if ( + averageNumMovesByPiece[uas].avgNumMoves === highestAverageMoves + ) { + pieceHighestAverageMoves.push(uas as UASymbol); + } + } + return { highestAverageMoves, pieceHighestAverageMoves }; +} + +function kdRatioMetrics( + KDMap: {}, + KDRatios: {}, + KDValuesMap: {}, + KDRatiosValues: {}, + maxKDRatio: number, + pieceWithHighestKDRatio: any[], + maxKDRatioValues: number, + pieceWithHighestKDRatioValues: any[] +) { + for (const uas of Object.keys(KDMap)) { + const kills = KDMap[uas].kills; + const deaths = KDMap[uas].deaths || 0; + if (deaths !== 0) { + KDRatios[uas] = kills / deaths; + } + } + for (const uas of Object.keys(KDValuesMap)) { + const valueKills = KDValuesMap[uas].valueKills; + const deaths = KDValuesMap[uas].deaths || 0; + if (deaths !== 0) { + KDRatiosValues[uas] = valueKills / deaths; + } + } + for (const uas of Object.keys(KDRatios)) { + if (KDRatios[uas] > maxKDRatio) { + maxKDRatio = KDRatios[uas]; + pieceWithHighestKDRatio = [uas as UASymbol]; + } else if (KDRatios[uas] === maxKDRatio) { + pieceWithHighestKDRatio.push(uas as UASymbol); // tie, add to the array + } + } + for (const uas of Object.keys(KDRatiosValues)) { + if (KDRatiosValues[uas] > maxKDRatioValues) { + maxKDRatioValues = KDRatiosValues[uas]; + pieceWithHighestKDRatioValues = [uas as UASymbol]; + } else if (KDRatiosValues[uas] === maxKDRatio) { + pieceWithHighestKDRatioValues.push(uas as UASymbol); // tie, add to the array + } + } + return { + maxKDRatio, + pieceWithHighestKDRatio, + maxKDRatioValues, + pieceWithHighestKDRatioValues, + }; +} + +function newFunction( + analysis: unknown, + totalMovesByPiece: {}, + singleGameMaxMoves: number, + pieceSingleGameMaxMoves: any[], + gameSingleGameMaxMoves: any[], + gamesNoCastling: number, + queenKingCastlingCounts: { + blackKing: number; + blackQueen: number; + whiteKing: number; + whiteQueen: number; + } +) { + const thisTotalMovesByPiece = + analysis['PieceLevelMoveInfoMetric']['totalMovesByPiece']; + for (const uas in thisTotalMovesByPiece) { + if (!totalMovesByPiece[uas]) { + totalMovesByPiece[uas] = { + numMoves: thisTotalMovesByPiece[uas].numMoves, + }; + } + totalMovesByPiece[uas].numMoves += thisTotalMovesByPiece[uas].numMoves; + } + + const thisSingleGameMaxMoves = + analysis['PieceLevelMoveInfoMetric']['uasSingleGameMaxMoves']; + const thisPieceSingleGameMaxMoves = + analysis['PieceLevelMoveInfoMetric']['uasWithMostMovesSingleGame']; + const thisGameSingleGameMaxMoves = + analysis['PieceLevelMoveInfoMetric']['gamesWithUasMostMoves']; + if (thisSingleGameMaxMoves > singleGameMaxMoves) { + singleGameMaxMoves = thisSingleGameMaxMoves; + pieceSingleGameMaxMoves = [thisPieceSingleGameMaxMoves as UASymbol]; + gameSingleGameMaxMoves = [thisGameSingleGameMaxMoves]; + } else if (thisSingleGameMaxMoves === singleGameMaxMoves) { + pieceSingleGameMaxMoves.push(thisPieceSingleGameMaxMoves as UASymbol); + gameSingleGameMaxMoves.push(thisGameSingleGameMaxMoves); + } + + const thisGamesNoCastling = + analysis['PieceLevelMoveInfoMetric']['gamesWithNoCastling']; + gamesNoCastling += thisGamesNoCastling; + + const thisQueenKingCastlingCounts = + analysis['PieceLevelMoveInfoMetric']['queenKingCastlingCounts']; + for (const count in thisQueenKingCastlingCounts) { + queenKingCastlingCounts[count] += thisQueenKingCastlingCounts[count]; + } + return { + singleGameMaxMoves, + pieceSingleGameMaxMoves, + gameSingleGameMaxMoves, + gamesNoCastling, + }; +} + +function aggregateRatingDiff( + analysis: unknown, + largestRatingDiff: number, + largestRatingDiffGame: any[] +) { + const thisLargestRatingDiff = analysis['MetadataMetric']['largestRatingDiff']; + const thisLargestRatingDiffGame = + analysis['MetadataMetric']['largestRatingDiffGame']; + if (thisLargestRatingDiff > largestRatingDiff) { + largestRatingDiff = thisLargestRatingDiff; + largestRatingDiffGame = thisLargestRatingDiffGame; + } else if (thisLargestRatingDiff === largestRatingDiff) { + largestRatingDiffGame.push(thisLargestRatingDiffGame); + } + return { largestRatingDiff, largestRatingDiffGame }; +} + +function aggregateDistanceMetrics( + analysis: unknown, + maxAvgDistance: number, + pieceMaxAvgDist: any[], + minAvgDistance: number, + pieceMinAvgDist: any[], + distPieceMaxDist: number, + pieceMaxDistSingleGame: any[], + gamePieceMaxDist: any[], + totalCollectiveDistGames: number, + gameMaxCollectiveDist: { distance: number; games: any[] }, + totalDistByPiece: {}, + avgDistByPiece: {} +) { + const thisMaxAvgDistance = analysis['MoveDistanceMetric']['maxAvgDistance']; + const thisPieceMaxAvgDistance = + analysis['MoveDistanceMetric']['pieceWithHighestAvg']; + if (thisMaxAvgDistance > maxAvgDistance) { + maxAvgDistance = thisMaxAvgDistance; + pieceMaxAvgDist = thisPieceMaxAvgDistance; + } else if (thisMaxAvgDistance === maxAvgDistance) { + pieceMaxAvgDist.push(thisPieceMaxAvgDistance); + } + + const thisMinAvgDistance = analysis['MoveDistanceMetric']['minAvgDistance']; + const thisPieceMinAvgDistance = + analysis['MoveDistanceMetric']['pieceWithLowestAvg']; + if (thisMinAvgDistance < minAvgDistance) { + minAvgDistance = thisMinAvgDistance; + pieceMinAvgDist = thisPieceMinAvgDistance; + } else if (thisMinAvgDistance === minAvgDistance) { + pieceMinAvgDist.push(thisPieceMinAvgDistance); + } + + const thisDistPieceMaxDist = + analysis['MoveDistanceMetric']['distanceThatPieceMovedInTheGame']; + const thisPieceMaxDistSingleGame = + analysis['MoveDistanceMetric']['pieceThatMovedTheFurthest']; + const thisGamePieceMaxDist = + analysis['MoveDistanceMetric']['gameInWhichPieceMovedTheFurthest']; + if (thisDistPieceMaxDist > distPieceMaxDist) { + distPieceMaxDist = thisDistPieceMaxDist; + pieceMaxDistSingleGame = thisPieceMaxDistSingleGame; + gamePieceMaxDist = thisGamePieceMaxDist; + } else if (thisDistPieceMaxDist === distPieceMaxDist) { + pieceMaxDistSingleGame.push(thisPieceMaxDistSingleGame); + gamePieceMaxDist.push(thisGamePieceMaxDist); + } + + const thisTotalCollectiveDistance = + analysis['MoveDistanceMetric']['totalCollectiveDistance']; + totalCollectiveDistGames += thisTotalCollectiveDistance; + + const thisGameMaxCollectiveDistance = + analysis['MoveDistanceMetric']['gameMaxCollectiveDistance']; + if (thisGameMaxCollectiveDistance.distance > gameMaxCollectiveDist.distance) { + gameMaxCollectiveDist = { + distance: thisGameMaxCollectiveDistance.distance, + games: [thisGameMaxCollectiveDistance.linkArray], + }; + } else if ( + thisGameMaxCollectiveDistance.distance === gameMaxCollectiveDist.distance + ) { + gameMaxCollectiveDist.games.push(thisGameMaxCollectiveDistance.linkArray); + } + + const thisTotalDistByPiece = + analysis['MoveDistanceMetric']['totalDistancesByPiece']; + for (const uas in thisTotalDistByPiece) { + if (!totalDistByPiece[uas]) { + totalDistByPiece[uas] = { + distance: thisTotalDistByPiece[uas].distance, + }; + } + totalDistByPiece[uas].distance += thisTotalDistByPiece[uas].distance; + } + + const thisAvgDistByPiece = + analysis['MoveDistanceMetric']['avgDistancesByPiece']; + for (const uas in thisAvgDistByPiece) { + if (!avgDistByPiece[uas]) { + avgDistByPiece[uas] = { + avgDistance: thisAvgDistByPiece[uas].avgDistance, + }; + } + avgDistByPiece[uas].avgDistance += thisAvgDistByPiece[uas].avgDistance; + } + return { + maxAvgDistance, + pieceMaxAvgDist, + minAvgDistance, + pieceMinAvgDist, + distPieceMaxDist, + pieceMaxDistSingleGame, + gamePieceMaxDist, + totalCollectiveDistGames, + gameMaxCollectiveDist, + }; +} + +function aggregateKillStreaks( + analysis: unknown, + KillStreakMap: {}, + maxKillStreak: number, + maxKillStreakPiece: any[], + maxKillStreakGame: any[] +) { + const thisKillStreakMap = analysis['KillStreakMetric']['killStreakMap']; + const thisMaxKillStreak = analysis['KillStreakMetric']['maxKillStreak']; + const thisMaxKillStreakPiece = + analysis['KillStreakMetric']['maxKillStreakPiece']; + const thisMaxKillStreakGame = + analysis['KillStreakMetric']['maxKillStreakGame']; + for (const uas in thisKillStreakMap) { + if (!KillStreakMap[uas]) { + KillStreakMap[uas] = 0; + } + if (thisKillStreakMap[uas].killStreaks > KillStreakMap[uas]) { + KillStreakMap[uas] = thisKillStreakMap[uas].killStreaks; + } + } + + // find maxes + if (thisMaxKillStreak > maxKillStreak) { + maxKillStreak = thisMaxKillStreak; + maxKillStreakPiece = thisMaxKillStreakPiece; + maxKillStreakGame = thisMaxKillStreakGame; + } else if (thisMaxKillStreak === maxKillStreak) { + maxKillStreakPiece.push(thisMaxKillStreakPiece); + maxKillStreakGame.push(thisMaxKillStreakGame); + } + return { maxKillStreak, maxKillStreakPiece, maxKillStreakGame }; +} + +function aggregateKDRatio(analysis: unknown, KDMap: {}, KDValuesMap: {}) { + const thisKDMap = analysis['KDRatioMetric']['KDMap']; + const thisKDValuesMap = analysis['KDRatioMetric']['KDValuesMap']; + + for (const uas in thisKDMap) { + if (!KDMap[uas]) { + KDMap[uas] = { + kills: 0, + deaths: 0, + revengeKills: 0, + }; + } + if (!KDValuesMap[uas]) { + KDValuesMap[uas] = { + valueKills: 0, + deaths: 0, + }; + } + KDMap[uas].kills += thisKDMap[uas].kills; + KDMap[uas].deaths += thisKDMap[uas].deaths; + KDMap[uas].revengeKills += thisKDMap[uas].revengeKills; + KDValuesMap[uas].valueKills += thisKDValuesMap[uas].valueKills; + KDValuesMap[uas].deaths += thisKDValuesMap[uas].deaths; + } +} + +function aggregateEndingStats(analysis: unknown, gameEndings: {}) { + const thisGameEndingsStats = analysis['MetadataMetric']['gameEndings']; + for (const ending in thisGameEndingsStats) { + if (!gameEndings[ending]) { + gameEndings[ending] = 0; + } + gameEndings[ending] += thisGameEndingsStats[ending]; + } +} + +function aggregateOpeningStats( + analysis: unknown, + openings: {}, + bongcloudAppearances: number +) { + const thisOpenings = analysis['MetadataMetric']['openings']; + for (const opening in thisOpenings) { + if (!openings[opening]) { + openings[opening] = { + appearances: 0, + blackWins: 0, + whiteWins: 0, + ties: 0, + whiteToBlackWinRatio: 0, + }; + } + openings[opening].appearances += thisOpenings[opening].appearances; + openings[opening].blackWins += thisOpenings[opening].blackWins; + openings[opening].whiteWins += thisOpenings[opening].whiteWins; + openings[opening].ties += thisOpenings[opening].ties; + // ratio accounting for ties + openings[opening].whiteToBlackWinRatio = + (openings[opening].whiteWins + openings[opening].ties) / + (openings[opening].blackWins + openings[opening].ties); + } + + bongcloudAppearances += analysis['MetadataMetric']['bongcloudAppearances']; + return bongcloudAppearances; +} + +function aggregateGameTimeControlStats( + analysis: unknown, + gameTimeControlStats: {} +) { + const thisGameTimeControlStats = + analysis['MetadataMetric']['gameTimeControlStats']; + for (const timeControl in thisGameTimeControlStats) { + if (!gameTimeControlStats[timeControl]) { + gameTimeControlStats[timeControl] = 0; + } + gameTimeControlStats[timeControl] += thisGameTimeControlStats[timeControl]; + } +} + +function aggregateMetadata( + analysis: unknown, + weightedTotalPlayerRating: number, + weightedTotalRatingDiff: number, + totalGamesAnalyzedForRatings: number +) { + const thisGamesAnalyzedForRatings = + analysis['MetadataMetric']['numberGamesAnalyzedForRatings']; + + const averagePlayerRating = analysis['MetadataMetric']['averagePlayerRating']; + weightedTotalPlayerRating += + averagePlayerRating * thisGamesAnalyzedForRatings; + + const averageRatingDiff = analysis['MetadataMetric']['averageRatingDiff']; + weightedTotalRatingDiff += averageRatingDiff * thisGamesAnalyzedForRatings; + + totalGamesAnalyzedForRatings += thisGamesAnalyzedForRatings; + return { + weightedTotalPlayerRating, + weightedTotalRatingDiff, + totalGamesAnalyzedForRatings, + }; +} + +function aggregatePromotions( + analysis: unknown, + promotedToTotals: { q: number; r: number; b: number; n: number }, + uasPromotingPieces: {} +) { + const thisPromotedToTotals = analysis['PromotionMetric']['promotedToTotals']; + promotedToTotals.q += thisPromotedToTotals.q; + promotedToTotals.r += thisPromotedToTotals.r; + promotedToTotals.b += thisPromotedToTotals.b; + promotedToTotals.n += thisPromotedToTotals.n; + + const thisUASPromotiongPieces = + analysis['PromotionMetric']['uasPromotingPieces']; + for (const uas in thisUASPromotiongPieces) { + if (!uasPromotingPieces[uas]) { + uasPromotingPieces[uas] = { + q: 0, + r: 0, + b: 0, + n: 0, + }; + } + uasPromotingPieces[uas].q += thisUASPromotiongPieces[uas].q; + uasPromotingPieces[uas].r += thisUASPromotiongPieces[uas].r; + uasPromotingPieces[uas].b += thisUASPromotiongPieces[uas].b; + uasPromotingPieces[uas].n += thisUASPromotiongPieces[uas].n; + } + const thisMaxNumQueens = analysis['PromotionMetric']['maxNumQueens']; + const thisMovesAndGamesMaxQueens = + analysis['PromotionMetric']['movesAndGamesWithMaxQueenCount']; + return { thisMaxNumQueens, thisMovesAndGamesMaxQueens }; +} + +function aggregateMatesAndAssists( + analysis: unknown, + mateAndAssistMap: {}, + matedCountsMap: { k: number; K: number } +) { + const thisMateAndAssistMap = + analysis['MateAndAssistMetric']['mateAndAssistMap']; + for (const uas in thisMateAndAssistMap) { + if (!mateAndAssistMap[uas]) { + mateAndAssistMap[uas] = { + mates: 0, + assists: 0, + hockeyAssists: 0, + }; + } + mateAndAssistMap[uas].mates += thisMateAndAssistMap[uas].mates; + mateAndAssistMap[uas].assists += thisMateAndAssistMap[uas].assists; + mateAndAssistMap[uas].hockeyAssists += + thisMateAndAssistMap[uas].hockeyAssists; + } + + const thisMatedCountsMap = analysis['MateAndAssistMetric']['matedCounts']; + matedCountsMap.k += thisMatedCountsMap.k; + matedCountsMap.K += thisMatedCountsMap.K; +} + +function aggregateGameTypeStats(analysis: unknown, gameTypeStats: {}) { + const thisGameTypeStats = analysis['MetadataMetric']['gameTypeStats']; + for (const gameType in thisGameTypeStats) { + if (!gameTypeStats[gameType]) { + gameTypeStats[gameType] = 0; + } + gameTypeStats[gameType] += thisGameTypeStats[gameType]; + } +} +// async function processAndAggregate() { +// await processFiles(); // This will wait until processFiles() is done +// await aggregateResults('src/results.json'); +// console.log('Final analysis complete.') +// } + +// processAndAggregate(); + +// export default aggregateResults; + +if (require.main === module) { + console.time('Total Final Analysis Execution Time'); + aggregateResults('src/results.json'); + console.timeEnd('Total Final Analysis Execution Time'); +} diff --git a/src/fileReader.ts b/src/fileReader.ts index e853f1c..de8f3eb 100644 --- a/src/fileReader.ts +++ b/src/fileReader.ts @@ -2,7 +2,10 @@ import { createReadStream } from 'fs'; import { createInterface } from 'readline'; import { FileReaderGame } from './types'; -// this should yield/stream a single game at a time as long as the game is complete +/** + * this should yield/stream a single game at a time as long as the game is complete + * @param path + */ export async function* gameChunks( path: string ): AsyncGenerator { @@ -27,7 +30,11 @@ export async function* gameChunks( } else if (line.startsWith('1.') && !ignoreGame) { moves = line; // Check if the moves line contains the game result - if (moves.includes('1-0') || moves.includes('0-1') || moves.includes('1/2-1/2')) { + if ( + moves.includes('1-0') || + moves.includes('0-1') || + moves.includes('1/2-1/2') + ) { // Check if both the metadata and moves are not empty before yielding the game if (metadata.length > 0 && moves.trim() !== '') { yield { diff --git a/src/final_aggregate_analysis.ts b/src/final_aggregate_analysis.ts deleted file mode 100644 index 9363a97..0000000 --- a/src/final_aggregate_analysis.ts +++ /dev/null @@ -1,625 +0,0 @@ -import * as fs from 'fs'; -import * as util from 'util'; -import { - ALL_SQUARES, - Piece, - PrettyMove, - Square, - UASymbol, -} from '../cjsmin/src/chess'; -// const processFiles = require('./streaming_partial_decompresser.js'); - -const readFile = util.promisify(fs.readFile); - -/** - * - * @param results.json - * @returns final analysis results of all files created and deleted in streaming_partial_decompresser - */ -async function aggregateResults(filePath: string) { - const data = JSON.parse(await readFile(filePath, 'utf-8')); - - // instantiate final variables - let totalGamesAnalyzed = 0; - let analysisCounter = 0; - - // metadata metrics - let largestRatingDiff = 0; - let largestRatingDiffGame = []; - let mostGamesPlayedByPlayer = 0; - let playerMostGames = []; - let gameTypeStats = {}; - let gameTimeControlStats = {}; - let openings = {}; - let bongcloudAppearances = 0; - let gameEndings = {}; - let totalGamesAnalyzedForRatings = 0; // accounting for some missing rating data - - // capture metrics - let KDMap = {}; - let KDValuesMap = {}; - let KDRatios = {}; - let KDRatiosValues = {} - let maxKDRatio = 0; - let maxKDRatioValues = 0; - let pieceWithHighestKDRatio = []; - let pieceWithHighestKDRatioValues = []; - let KillStreakMap = {}; - let maxKillStreak = 0; - let maxKillStreakPiece = []; - let maxKillStreakGame = []; - - // mates and assists metrics - let mateAndAssistMap = {}; - let matedCountsMap = { - k: 0, - K: 0 - }; - - // promotions metrics - let promotedToTotals = { - q: 0, - r: 0, - b: 0, - n: 0, - } - let uasPromotingPieces = {}; - let maxNumQueens = 0; - let movesAndGamesMaxQueens = []; - - // distance metrics - let pieceMaxAvgDist = []; - let maxAvgDistance = 0; - let pieceMinAvgDist = []; - let minAvgDistance = Infinity; - let pieceMaxDistSingleGame = []; - let gamePieceMaxDist = []; - let distPieceMaxDist = 0; - let totalCollectiveDistGames = 0; - let gameMaxCollectiveDist = { - distance: 0, - games: [], - }; - let totalDistByPiece = {}; - let avgDistByPiece = {}; - - // moves metrics - let gameMostMoves = []; - let gameMostMovesNumMoves = 0; - let totalMovesByPiece = {}; - let averageNumMovesByPiece = {}; - let pieceHighestAverageMoves = []; - let highestAverageMoves = 0; - let singleGameMaxMoves = 0; - let pieceSingleGameMaxMoves = []; - let gameSingleGameMaxMoves = []; - let gamesNoCastling = 0; - let queenKingCastlingCounts = { - blackKing: 0, - blackQueen: 0, - whiteKing: 0, - whiteQueen: 0, - }; - let enPassantMovesCount = 0; - let totalNumPiecesKnightHopped = 0; - - - // helper variables - let weightedTotalPlayerRating = 0; - let weightedTotalRatingDiff = 0; - - // ANALYSIS-BY-ANALYSIS CALCULATIONS - for (const analysis of Object.values(data)) { - analysisCounter++; - - const thisAnalysisGamesAnalyzed = analysis['Number of games analyzed']; - - // METADATA METRICS - // ratings weighted average calculations - const thisGamesAnalyzedForRatings = analysis['MetadataMetric']['numberGamesAnalyzedForRatings']; - - const averagePlayerRating = analysis['MetadataMetric']['averagePlayerRating']; - weightedTotalPlayerRating += averagePlayerRating * thisGamesAnalyzedForRatings; - - const averageRatingDiff = analysis['MetadataMetric']['averageRatingDiff']; - weightedTotalRatingDiff += averageRatingDiff * thisGamesAnalyzedForRatings; - - totalGamesAnalyzedForRatings += thisGamesAnalyzedForRatings; - - // ratings largest diff - const thisLargestRatingDiff = analysis['MetadataMetric']['largestRatingDiff']; - const thisLargestRatingDiffGame = analysis['MetadataMetric']['largestRatingDiffGame']; - if (thisLargestRatingDiff > largestRatingDiff) { - largestRatingDiff = thisLargestRatingDiff; - largestRatingDiffGame = thisLargestRatingDiffGame; - } else if (thisLargestRatingDiff === largestRatingDiff) { - largestRatingDiffGame.push(thisLargestRatingDiffGame); - } - - // games played - // currently this stat is inaccurately tracked across analyses - // the player with the most games might have their games split across analyses - // the fix is to return the playerGameStats from misc.ts and then parse that in the final aggregate analysis but that dataset could be very large - const thisMostGamesPlayed = analysis['MetadataMetric']['mostGamesPlayed']; - const thisPlayerMostGames = analysis['MetadataMetric']['playerMostGames']; - if (thisMostGamesPlayed > mostGamesPlayedByPlayer) { - mostGamesPlayedByPlayer = thisMostGamesPlayed; - playerMostGames = thisPlayerMostGames; - } else if (thisMostGamesPlayed === mostGamesPlayedByPlayer) { - playerMostGames.push(thisPlayerMostGames); - } - - // aggregate gameTypeStats - const thisGameTypeStats = analysis['MetadataMetric']['gameTypeStats']; - for (const gameType in thisGameTypeStats) { - if (!gameTypeStats[gameType]) { - gameTypeStats[gameType] = 0; - } - gameTypeStats[gameType] += thisGameTypeStats[gameType]; - } - - // aggregate gameTimeControlStats - const thisGameTimeControlStats = analysis['MetadataMetric']['gameTimeControlStats']; - for (const timeControl in thisGameTimeControlStats) { - if (!gameTimeControlStats[timeControl]) { - gameTimeControlStats[timeControl] = 0; - } - gameTimeControlStats[timeControl] += thisGameTimeControlStats[timeControl]; - } - - // aggregate openings stats - const thisOpenings = analysis['MetadataMetric']['openings']; - for (const opening in thisOpenings) { - if (!openings[opening]) { - openings[opening] = { - appearances: 0, - blackWins: 0, - whiteWins: 0, - ties: 0, - whiteToBlackWinRatio: 0, - }; - } - openings[opening].appearances += thisOpenings[opening].appearances; - openings[opening].blackWins += thisOpenings[opening].blackWins; - openings[opening].whiteWins += thisOpenings[opening].whiteWins; - openings[opening].ties += thisOpenings[opening].ties; - // ratio accounting for ties - openings[opening].whiteToBlackWinRatio = (openings[opening].whiteWins + openings[opening].ties) / (openings[opening].blackWins + openings[opening].ties); - } - - bongcloudAppearances += analysis['MetadataMetric']['bongcloudAppearances']; - - // aggregate game endings stats - const thisGameEndingsStats = analysis['MetadataMetric']['gameEndings']; - for (const ending in thisGameEndingsStats) { - if (!gameEndings[ending]) { - gameEndings[ending] = 0; - } - gameEndings[ending] += thisGameEndingsStats[ending]; - } - - // KD AND CAPTURE METRICS - // KD Ratios - // Recalculating KD Ratios across all the analyses (alternatively could do weighted averages) - const thisKDMap = analysis['KDRatioMetric']['KDMap'] - const thisKDValuesMap = analysis['KDRatioMetric']['KDValuesMap'] - - for (const uas in thisKDMap) { - if (!KDMap[uas]) { - KDMap[uas] = { - kills: 0, - deaths: 0, - revengeKills: 0 - }; - } - if (!KDValuesMap[uas]) { - KDValuesMap[uas] = { - valueKills: 0, - deaths: 0 - }; - } - KDMap[uas].kills += thisKDMap[uas].kills; - KDMap[uas].deaths += thisKDMap[uas].deaths; - KDMap[uas].revengeKills += thisKDMap[uas].revengeKills; - KDValuesMap[uas].valueKills += thisKDValuesMap[uas].valueKills; - KDValuesMap[uas].deaths += thisKDValuesMap[uas].deaths; - } - - // kill streaks - const thisKillStreakMap = analysis['KillStreakMetric']['killStreakMap'] - const thisMaxKillStreak = analysis['KillStreakMetric']['maxKillStreak'] - const thisMaxKillStreakPiece = analysis['KillStreakMetric']['maxKillStreakPiece'] - const thisMaxKillStreakGame = analysis['KillStreakMetric']['maxKillStreakGame'] - for (const uas in thisKillStreakMap) { - if (!KillStreakMap[uas]) { - KillStreakMap[uas] = 0; - } - if (thisKillStreakMap[uas].killStreaks > KillStreakMap[uas]) { - KillStreakMap[uas] = thisKillStreakMap[uas].killStreaks; - } - } - // find maxes - if (thisMaxKillStreak > maxKillStreak) { - maxKillStreak = thisMaxKillStreak; - maxKillStreakPiece = thisMaxKillStreakPiece; - maxKillStreakGame = thisMaxKillStreakGame; - } else if (thisMaxKillStreak === maxKillStreak) { - maxKillStreakPiece.push(thisMaxKillStreakPiece); - maxKillStreakGame.push(thisMaxKillStreakGame); - } - - // mates and assists - const thisMateAndAssistMap = analysis['MateAndAssistMetric']['mateAndAssistMap'] - for (const uas in thisMateAndAssistMap) { - if (!mateAndAssistMap[uas]) { - mateAndAssistMap[uas] = { - mates: 0, - assists: 0, - hockeyAssists: 0 - }; - } - mateAndAssistMap[uas].mates += thisMateAndAssistMap[uas].mates; - mateAndAssistMap[uas].assists += thisMateAndAssistMap[uas].assists; - mateAndAssistMap[uas].hockeyAssists += thisMateAndAssistMap[uas].hockeyAssists; - } - - const thisMatedCountsMap = analysis['MateAndAssistMetric']['matedCounts'] - matedCountsMap.k += thisMatedCountsMap.k; - matedCountsMap.K += thisMatedCountsMap.K; - - // promotions metrics - const thisPromotedToTotals = analysis['PromotionMetric']['promotedToTotals'] - promotedToTotals.q += thisPromotedToTotals.q - promotedToTotals.r += thisPromotedToTotals.r - promotedToTotals.b += thisPromotedToTotals.b - promotedToTotals.n += thisPromotedToTotals.n - - const thisUASPromotiongPieces = analysis['PromotionMetric']['uasPromotingPieces'] - for (const uas in thisUASPromotiongPieces) { - if (!uasPromotingPieces[uas]) { - uasPromotingPieces[uas] = { - q: 0, - r: 0, - b: 0, - n: 0, - }; - } - uasPromotingPieces[uas].q += thisUASPromotiongPieces[uas].q - uasPromotingPieces[uas].r += thisUASPromotiongPieces[uas].r - uasPromotingPieces[uas].b += thisUASPromotiongPieces[uas].b - uasPromotingPieces[uas].n += thisUASPromotiongPieces[uas].n - } - const thisMaxNumQueens = analysis['PromotionMetric']['maxNumQueens'] - const thisMovesAndGamesMaxQueens = analysis['PromotionMetric']['movesAndGamesWithMaxQueenCount'] - - // find maxes - if (thisMaxNumQueens > maxNumQueens) { - maxNumQueens = thisMaxNumQueens; - movesAndGamesMaxQueens = thisMovesAndGamesMaxQueens; - } else if (thisMaxNumQueens > maxNumQueens) { - movesAndGamesMaxQueens.push(thisMovesAndGamesMaxQueens); - } - - // distance metrics - const thisMaxAvgDistance = analysis['MoveDistanceMetric']['maxAvgDistance'] - const thisPieceMaxAvgDistance = analysis['MoveDistanceMetric']['pieceWithHighestAvg'] - if (thisMaxAvgDistance > maxAvgDistance) { - maxAvgDistance = thisMaxAvgDistance - pieceMaxAvgDist = thisPieceMaxAvgDistance - } else if (thisMaxAvgDistance === maxAvgDistance) { - pieceMaxAvgDist.push(thisPieceMaxAvgDistance) - } - - const thisMinAvgDistance = analysis['MoveDistanceMetric']['minAvgDistance'] - const thisPieceMinAvgDistance = analysis['MoveDistanceMetric']['pieceWithLowestAvg'] - if (thisMinAvgDistance < minAvgDistance) { - minAvgDistance = thisMinAvgDistance - pieceMinAvgDist = thisPieceMinAvgDistance - } else if (thisMinAvgDistance === minAvgDistance) { - pieceMinAvgDist.push(thisPieceMinAvgDistance) - } - - const thisDistPieceMaxDist = analysis['MoveDistanceMetric']['distanceThatPieceMovedInTheGame'] - const thisPieceMaxDistSingleGame = analysis['MoveDistanceMetric']['pieceThatMovedTheFurthest'] - const thisGamePieceMaxDist = analysis['MoveDistanceMetric']['gameInWhichPieceMovedTheFurthest'] - if (thisDistPieceMaxDist > distPieceMaxDist) { - distPieceMaxDist = thisDistPieceMaxDist; - pieceMaxDistSingleGame = thisPieceMaxDistSingleGame; - gamePieceMaxDist = thisGamePieceMaxDist; - } else if (thisDistPieceMaxDist === distPieceMaxDist) { - pieceMaxDistSingleGame.push(thisPieceMaxDistSingleGame); - gamePieceMaxDist.push(thisGamePieceMaxDist); - } - - const thisTotalCollectiveDistance = analysis['MoveDistanceMetric']['totalCollectiveDistance'] - totalCollectiveDistGames += thisTotalCollectiveDistance; - - const thisGameMaxCollectiveDistance = analysis['MoveDistanceMetric']['gameMaxCollectiveDistance'] - if (thisGameMaxCollectiveDistance.distance > gameMaxCollectiveDist.distance) { - gameMaxCollectiveDist = { - distance: thisGameMaxCollectiveDistance.distance, - games: [thisGameMaxCollectiveDistance.linkArray], - }; - } else if (thisGameMaxCollectiveDistance.distance === gameMaxCollectiveDist.distance) { - gameMaxCollectiveDist.games.push(thisGameMaxCollectiveDistance.linkArray); - } - - const thisTotalDistByPiece = analysis['MoveDistanceMetric']['totalDistancesByPiece'] - for (const uas in thisTotalDistByPiece) { - if (!totalDistByPiece[uas]) { - totalDistByPiece[uas] = { - distance: thisTotalDistByPiece[uas].distance - } - } - totalDistByPiece[uas].distance += thisTotalDistByPiece[uas].distance; - } - - const thisAvgDistByPiece = analysis['MoveDistanceMetric']['avgDistancesByPiece'] - for (const uas in thisAvgDistByPiece) { - if (!avgDistByPiece[uas]) { - avgDistByPiece[uas] = { - avgDistance: thisAvgDistByPiece[uas].avgDistance - } - } - avgDistByPiece[uas].avgDistance += thisAvgDistByPiece[uas].avgDistance; - } - - // moves metrics - const thisGameMostMoves = analysis['GameWithMostMovesMetric']['gameWithMostMoves']; - const thisGameMostMovesNumMoves = analysis['GameWithMostMovesMetric']['gameWithMostMovesNumMoves']; - if (thisGameMostMovesNumMoves > gameMostMovesNumMoves) { - gameMostMovesNumMoves = thisGameMostMovesNumMoves; - gameMostMoves = [thisGameMostMoves]; - } else if (thisGameMostMovesNumMoves === gameMostMovesNumMoves) { - gameMostMoves.push(thisGameMostMoves); - } - - // piece level moves metrics - const thisTotalMovesByPiece = analysis['PieceLevelMoveInfoMetric']['totalMovesByPiece'] - for (const uas in thisTotalMovesByPiece) { - if (!totalMovesByPiece[uas]) { - totalMovesByPiece[uas] = { - numMoves: thisTotalMovesByPiece[uas].numMoves - } - } - totalMovesByPiece[uas].numMoves += thisTotalMovesByPiece[uas].numMoves; - } - - const thisSingleGameMaxMoves = analysis['PieceLevelMoveInfoMetric']['uasSingleGameMaxMoves'] - const thisPieceSingleGameMaxMoves = analysis['PieceLevelMoveInfoMetric']['uasWithMostMovesSingleGame'] - const thisGameSingleGameMaxMoves = analysis['PieceLevelMoveInfoMetric']['gamesWithUasMostMoves'] - if (thisSingleGameMaxMoves > singleGameMaxMoves) { - singleGameMaxMoves = thisSingleGameMaxMoves; - pieceSingleGameMaxMoves = [thisPieceSingleGameMaxMoves as UASymbol] - gameSingleGameMaxMoves = [thisGameSingleGameMaxMoves] - } else if (thisSingleGameMaxMoves === singleGameMaxMoves) { - pieceSingleGameMaxMoves.push(thisPieceSingleGameMaxMoves as UASymbol) - gameSingleGameMaxMoves.push(thisGameSingleGameMaxMoves) - } - - const thisGamesNoCastling = analysis['PieceLevelMoveInfoMetric']['gamesWithNoCastling'] - gamesNoCastling += thisGamesNoCastling; - - const thisQueenKingCastlingCounts = analysis['PieceLevelMoveInfoMetric']['queenKingCastlingCounts']; - for (const count in thisQueenKingCastlingCounts) { - queenKingCastlingCounts[count] += thisQueenKingCastlingCounts[count]; - } - - // misc move fact metrics - const thisEnPassantMovesCount = analysis['MiscMoveFactMetric']['enPassantMovesCount']; - enPassantMovesCount += thisEnPassantMovesCount; - - const thisTotalNumPiecesKnightHopped = analysis['MiscMoveFactMetric']['totalNumPiecesKnightHopped']; - totalNumPiecesKnightHopped += thisTotalNumPiecesKnightHopped; - - - // final increments - totalGamesAnalyzed += thisAnalysisGamesAnalyzed; - } - - // AGGREGATE CALCULATIONS - // ratings weighted average calculations - const weightedAveragePlayerRating = weightedTotalPlayerRating / totalGamesAnalyzedForRatings; - const weightedAverageRatingDiff = weightedTotalRatingDiff / totalGamesAnalyzedForRatings - - // calculating KD Ratios and maxes for final maps - for (const uas of Object.keys(KDMap)) { - const kills = KDMap[uas].kills; - const deaths = KDMap[uas].deaths || 0; - if (deaths !== 0) { - KDRatios[uas] = kills / deaths; - } - } - for (const uas of Object.keys(KDValuesMap)) { - const valueKills = KDValuesMap[uas].valueKills; - const deaths = KDValuesMap[uas].deaths || 0; - if (deaths !== 0) { - KDRatiosValues[uas] = valueKills / deaths; - } - } - for (const uas of Object.keys(KDRatios)) { - if (KDRatios[uas] > maxKDRatio) { - maxKDRatio = KDRatios[uas]; - pieceWithHighestKDRatio = [uas as UASymbol]; - } else if (KDRatios[uas] === maxKDRatio) { - pieceWithHighestKDRatio.push(uas as UASymbol); // tie, add to the array - } - } - for (const uas of Object.keys(KDRatiosValues)) { - if (KDRatiosValues[uas] > maxKDRatioValues) { - maxKDRatioValues = KDRatiosValues[uas]; - pieceWithHighestKDRatioValues = [uas as UASymbol]; - } else if (KDRatiosValues[uas] === maxKDRatio) { - pieceWithHighestKDRatioValues.push(uas as UASymbol); // tie, add to the array - } - } - - // calculating averageNumMovesByPiece (without doing weighted averages) and related maxes - for (const uas in totalMovesByPiece) { - if (!averageNumMovesByPiece[uas]) { - averageNumMovesByPiece[uas] = { - avgNumMoves: totalMovesByPiece[uas].numMoves / totalGamesAnalyzed - } - } - } - - for (const uas in averageNumMovesByPiece) { - if (averageNumMovesByPiece[uas].avgNumMoves > highestAverageMoves) { - highestAverageMoves = averageNumMovesByPiece[uas].avgNumMoves; - pieceHighestAverageMoves = [uas as UASymbol]; - } else if (averageNumMovesByPiece[uas].avgNumMoves === highestAverageMoves) { - pieceHighestAverageMoves.push(uas as UASymbol) - } - } - - // LOGS FOR THE ENTIRE SET - // metadata logs - console.log('GAME SET STATS (METADATA) ----------------------------'); - console.log(`Average Player Rating: ${weightedAveragePlayerRating}`); - console.log(`Average Rating Difference: ${weightedAverageRatingDiff}`); - console.log(`Largest Rating Difference: ${largestRatingDiff}`); - console.log(`Largest Rating Difference Game(s): ${largestRatingDiffGame}`); - console.log(`Player(s) with the most games played (CURRENTLY INACCURATELY TRACKED): ${playerMostGames}`); - console.log(`Number of games played (CURRENTLY INACCURATELY TRACKED): ${mostGamesPlayedByPlayer}`); - console.log('\n'); - console.log(`Game Type Stats: `), - console.table(gameTypeStats); - console.log(`Time Control Stats: (filtered by appearing in at least 1% of games):`); - - const sortedGameTimeControlStats = Object.entries(gameTimeControlStats) - .sort(([, valueA], [, valueB]) => Number(valueB) - Number(valueA)); - - const filteredGameTimeControlStats = Object.fromEntries( - sortedGameTimeControlStats - .filter(([_, value]) => (value as number) / totalGamesAnalyzed > 0.01) - ); - - console.table(filteredGameTimeControlStats); - - console.log('Openings stats (filtered by appearing in at least 1% of games):'); - - const sortedOpenings = Object.entries(openings) - .sort(([, dataA]: [string, { whiteToBlackWinRatio: number | null; }], [, dataB]: [string, { whiteToBlackWinRatio: number | null; }]) => - (dataB.whiteToBlackWinRatio || 0) - (dataA.whiteToBlackWinRatio || 0) - ); - const filteredOpenings = Object.fromEntries( - sortedOpenings - .filter(([_, data]: [string, { appearances: number; blackWins: number; whiteWins: number; ties: number; whiteToBlackWinRatio: number | null; }]) => data.appearances / totalGamesAnalyzed > 0.01) - ); - - console.table(filteredOpenings); - - console.log(`Number of bongcloud appearances: ${bongcloudAppearances}`); - console.log(`Game Endings: `), - console.table(gameEndings); - console.log('\n'); - - - // captures logs - console.log('CAPTURES STATS: ----------------------------'); - console.log('Kills, deaths, and revenge kills for each unambiguous piece:'), - console.table(KDMap); - console.log( - 'Kill Death Ratios for each unambiguous piece: ' + - JSON.stringify(KDRatios, null, 2) - ); - console.log( - `Piece with the highest KD ratio was ${pieceWithHighestKDRatio} with a ratio of ${maxKDRatio}` - ); - - console.log('\n'); - console.log("Piece values for kills: Pawn 1 point, Knight 3 points, Bishop 3 points, Rook 5 points, Queen 9 points, King 4 points. ") - console.log('Value kills and deaths for each unambiguous piece:'), - console.table(KDValuesMap); - console.log( - 'Kill Death Ratios for each unambiguous piece: ' + - JSON.stringify(KDRatiosValues, null, 2) - ); - console.log( - `Piece with the highest KD ratio (taking into account piece values) was ${pieceWithHighestKDRatioValues} with a ratio of ${maxKDRatioValues}` - ); - - console.log('\n'); - console.log( - 'Max Kill Streaks achieved for each piece: '); - console.table(KillStreakMap) - console.log(`Max Kill Streak achieved by any piece (the number of captures without any other piece on its team capturing. doesn't have to be consecutive move captures): ${maxKillStreak} by the piece(s) ${maxKillStreakPiece}. This was done in the game(s): `); - console.log(maxKillStreakGame.join('\n')); - - // mates and assists logs - console.log('\n'); - console.log('MATES AND ASSISTS STATS: ----------------------------'); - console.log( - 'Mates, assists, and hockey assists for each piece: '); - console.table(mateAndAssistMap) - console.log('Note: any "mates" attributed to kings are a result of a king moving to reveal a discovered mate.') - console.log( - 'Number of times each king was mated: '); - console.table(matedCountsMap) - - // promotions logs - console.log('\n'); - console.log('PROMOTIONS STATS: ----------------------------'); - console.log( - 'Pieces promoted to most often: '); - console.table(promotedToTotals) - console.log( - 'The pieces each unambiguous piece promotes to most often: '); - console.table(uasPromotingPieces); - console.log(`The maximum number of queens to appear in a given move in a game: ${maxNumQueens}`); - console.log(`The games(s) and first move(s) in that game in which that number of queens appeared: - ${movesAndGamesMaxQueens.map(move => - JSON.stringify(move, null, 2)).join(", ")}`); - - // distance logs - console.log('\n'); - console.log('DISTANCE STATS: ----------------------------'); - console.log(`Piece(s) with highest average distance: ${pieceMaxAvgDist}. That/those piece(s) average distance: ${maxAvgDistance}`); - console.log(`Piece(s) with lowest average distance: ${pieceMinAvgDist}. That/those piece(s) average distance: ${minAvgDistance}`); - console.log(`Piece that covered the most ground in a single game: ${pieceMaxDistSingleGame}. Distance covered: ${distPieceMaxDist}. Game in which that distance was covered by that piece: ${gamePieceMaxDist}.`); - console.log(`Total collective distance of all pieces in games analyzed: ${totalCollectiveDistGames}`); - console.log(`Game(s) with the furthest collective distance moved: ${gameMaxCollectiveDist.games}`); - console.log(`Distance moved: ${gameMaxCollectiveDist.distance}`); - console.log(`Total distance moved by piece:`); - console.table(totalDistByPiece); - console.log(`Average distance moved by piece:`); - console.table(avgDistByPiece); - - // - console.log('\n'); - console.log('MOVES STATS: ----------------------------'); - console.log(`Game(s) with most moves made (1 move = one white or one black move): ${gameMostMoves}`); - console.log(`Number of moves made: ${gameMostMovesNumMoves}`); - console.log('Total number of moves made by each piece: ') - console.table(totalMovesByPiece) - console.log('Average number of moves made by each piece: ') - console.table(averageNumMovesByPiece) - console.log(`Piece(s) with the highest average number of moves: ${pieceHighestAverageMoves}. The average number of moves that/those pieces made per game: ${highestAverageMoves}`) - console.log(`The piece with the most moves played in a single game: ${pieceSingleGameMaxMoves}. The number of moves played in that game: ${singleGameMaxMoves}. The game it played that number of moves in: ${gameSingleGameMaxMoves}`) - console.log(`The number of games with no castling: ${gamesNoCastling}`) - console.log('The number of times each kind of castling happened: ') - console.table(queenKingCastlingCounts); - console.log(`The number of En passants that occured: ${enPassantMovesCount}`) - console.log(`The number of pieces that were hopped over by a knight: ${totalNumPiecesKnightHopped}`) - - // final analysis logs - console.log('\n'); - console.log('ANALYSIS STATS: ----------------------------') - console.log(`Total games analyzed: ${totalGamesAnalyzed}`); - console.log(`Number of separate analyses: ${analysisCounter}`) -} - -console.time('Total Final Analysis Execution Time'); -aggregateResults('src/results.json'); -console.timeEnd('Total Final Analysis Execution Time'); - - -// async function processAndAggregate() { -// await processFiles(); // This will wait until processFiles() is done -// await aggregateResults('src/results.json'); -// console.log('Final analysis complete.') -// } - -// processAndAggregate(); - -// export default aggregateResults; \ No newline at end of file diff --git a/src/index_with_decompressor.ts b/src/index_with_decompressor.ts deleted file mode 100644 index b89524b..0000000 --- a/src/index_with_decompressor.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { Chess } from '../cjsmin/src/chess'; -import { gameChunks } from './fileReader'; -import { KDRatioMetric, MateAndAssistMetric, KillStreakMetric } from './metrics/captures'; -import { MoveDistanceMetric } from './metrics/distances'; -import { MetadataMetric } from './metrics/misc'; -import { - GameWithMostMovesMetric, - PieceLevelMoveInfoMetric, - MiscMoveFactMetric, -} from './metrics/moves'; -import { PromotionMetric } from './metrics/promotions'; -import * as fs from 'fs'; -import * as path from 'path'; -import * as asyncLib from 'async'; - -/** - * - * @param path - * @returns - */ -export async function main(path: string) { - console.time('Total Execution Time'); - await gameIterator(path); - console.timeEnd('Total Execution Time'); - return results; -} - -let results = { - 'Number of games analyzed': 0, -} - -/** - * Metric functions will ingest a single game at a time - * @param metricFunctions - */ -async function gameIterator(path) { - const cjsmin = new Chess(); - - const gamesGenerator = gameChunks(path); - const kdRatioMetric = new KDRatioMetric(); - const killStreakMetric = new KillStreakMetric(); - const mateAndAssistMetric = new MateAndAssistMetric(); - const promotionMetric = new PromotionMetric(); - const moveDistanceMetric = new MoveDistanceMetric(); - const gameWithMostMovesMetric = new GameWithMostMovesMetric(); - const pieceLevelMoveInfoMetric = new PieceLevelMoveInfoMetric(); - const metadataMetric = new MetadataMetric(cjsmin); - const miscMoveFactMetric = new MiscMoveFactMetric(); - const metrics = [ - metadataMetric, - kdRatioMetric, - killStreakMetric, - mateAndAssistMetric, - promotionMetric, - moveDistanceMetric, - gameWithMostMovesMetric, - pieceLevelMoveInfoMetric, - miscMoveFactMetric, - ]; - - let gameCounter = 0; - for await (const { moves, metadata } of gamesGenerator) { - gameCounter++; - if (gameCounter % 400 == 0) { - console.log('number of games ingested: ', gameCounter); - } - - for (const metric of metrics) { - // with array creation - const historyGenerator = cjsmin.historyGeneratorArr(moves); - metric.processGame(Array.from(historyGenerator), metadata); - } - } - results['Number of games analyzed'] = gameCounter; - let metricCallsCount = 0; - for (const metric of metrics) { - metricCallsCount++; - results[metric.constructor.name] = metric.aggregate() - } -} - -// Create a write to result.json queue with a concurrency of 1 -const queue = asyncLib.queue((task) => { - return new Promise((resolve, reject) => { - const { results, analysisKey, resultsPath } = task; - try { - fs.writeFileSync(resultsPath, JSON.stringify(results, null, 2)); - console.log(`Analysis "${analysisKey}" has been written to ${resultsPath}`); - resolve(); - } catch (err) { - reject(err); - } - }); -}, 1); - -// for use with streaming_partial_decompresser.js -if (require.main === module) { - const pathToAnalyze = process.argv[2]; - main(pathToAnalyze).then(async (results) => { - const now = new Date(); - const milliseconds = now.getMilliseconds(); - - const analysisKey = `analysis_${now.toLocaleString().replace(/\/|,|:|\s/g, '_')}_${milliseconds}`; - const resultsPath = path.join(__dirname, 'results.json'); - - let existingResults = {}; - if (fs.existsSync(resultsPath)) { - const fileContent = fs.readFileSync(resultsPath, 'utf8'); - if (fileContent !== '') { - existingResults = JSON.parse(fileContent); - } - } - - existingResults[analysisKey] = results; - - // Add the write task to the queue and wait for it to complete - await queue.push({ results: existingResults, analysisKey, resultsPath }); - }); -} \ No newline at end of file diff --git a/src/metrics/misc.ts b/src/metrics/misc.ts index ba16aa1..a245357 100644 --- a/src/metrics/misc.ts +++ b/src/metrics/misc.ts @@ -1,4 +1,4 @@ -import { Piece, PrettyMove, Chess} from '../../cjsmin/src/chess'; +import { Chess, Piece, PrettyMove } from '../../cjsmin/src/chess'; import { Metric } from './metric'; // calculates how many games in the dataset @@ -35,13 +35,14 @@ export class MetadataMetric implements Metric { blackWins: number; whiteWins: number; ties: number; - openings: - { [opening: string]: - { appearances: number, - blackWins: number, - whiteWins: number, - ties: number, - whiteToBlackWinRatio: number } + openings: { + [opening: string]: { + appearances: number; + blackWins: number; + whiteWins: number; + ties: number; + whiteToBlackWinRatio: number; + }; }; bongcloud: number; @@ -98,15 +99,17 @@ export class MetadataMetric implements Metric { // Number of games played by time control type console.log('Number of games played by time control type:'); const filteredTimeControlStats = Object.fromEntries( - Object.entries(this.gameTimeControlStats) - .filter(([_, count]) => count / this.numberGamesAnalyzed > 0.05) + Object.entries(this.gameTimeControlStats).filter( + ([_, count]) => count / this.numberGamesAnalyzed > 0.05 + ) ); console.table(filteredTimeControlStats); // Openings by number of times they appear and their win rates console.log('Openings by number of times they appear and their win rates:'); const filteredOpenings = Object.fromEntries( - Object.entries(this.openings) - .filter(([_, data]) => data.appearances / this.numberGamesAnalyzed > 0.05) + Object.entries(this.openings).filter( + ([_, data]) => data.appearances / this.numberGamesAnalyzed > 0.05 + ) ); console.table(filteredOpenings); @@ -115,11 +118,10 @@ export class MetadataMetric implements Metric { // console.log('Openings by number of times they appear and their win rates: '), // console.table(this.openings); - console.log('Number of times bongcloud appeared: ', this.bongcloud) - console.log('\n') + console.log('Number of times bongcloud appeared: ', this.bongcloud); + console.log('\n'); - console.log('Game Endings: '), - console.table(this.gameEndings); + console.log('Game Endings: '), console.table(this.gameEndings); } // Reset the maps used to track metrics @@ -153,7 +155,7 @@ export class MetadataMetric implements Metric { processGame( game: { move: PrettyMove; board: Piece[] }[], - metadata?: string[], + metadata?: string[] ) { // Update the gameTimeControlStats based on the time control of the game const timeControl = metadata?.find((data) => @@ -212,10 +214,12 @@ export class MetadataMetric implements Metric { if (ratingDiff > this.largestRatingDiff) { this.largestRatingDiff = ratingDiff; this.largestRatingDiffGame = [ - metadata?.find((data) => data.startsWith('[Site')) || '']; + metadata?.find((data) => data.startsWith('[Site')) || '', + ]; } else if (ratingDiff === this.largestRatingDiff) { this.largestRatingDiffGame.push( - metadata?.find((data) => data.startsWith('[Site')) || ''); // tie, add to array + metadata?.find((data) => data.startsWith('[Site')) || '' + ); // tie, add to array } } @@ -236,7 +240,7 @@ export class MetadataMetric implements Metric { } // black vs white wins - let result = metadata.find(item => item.startsWith('[Result')); + let result = metadata.find((item) => item.startsWith('[Result')); if (result === '[Result "1-0"]') { this.whiteWins++; @@ -248,121 +252,132 @@ export class MetadataMetric implements Metric { console.log('Invalid result'); } -// extract openings from metadata -const opening = metadata?.find((item) => item.startsWith('[Opening "')) -?.replace('[Opening "', '') -?.replace('"]', ''); + // extract openings from metadata + const opening = metadata + ?.find((item) => item.startsWith('[Opening "')) + ?.replace('[Opening "', '') + ?.replace('"]', ''); -if(opening) { - if (opening.toLowerCase() == "bongcloud") { - this.bongcloud++; - } + if (opening) { + if (opening.toLowerCase() == 'bongcloud') { + this.bongcloud++; + } - // add opening to openings object - if (this.openings[opening]) { - this.openings[opening].appearances++; - switch(result) { - case '[Result "1-0"]': - this.openings[opening].whiteWins++; - break; - case '[Result "0-1"]': - this.openings[opening].blackWins++; - break; - case '[Result "1/2-1/2"]': - this.openings[opening].whiteWins += 0.5; - this.openings[opening].blackWins += 0.5; - this.openings[opening].ties++; - break; - } - this.openings[opening].whiteToBlackWinRatio = this.openings[opening].whiteWins / this.openings[opening].blackWins; - } else { - let blackWins, whiteWins, ties, whiteToBlackWinRatio; - - switch(result) { - case '[Result "0-1"]': - blackWins = 1; - whiteWins = 0; - ties = 0; - whiteToBlackWinRatio = 0; - break; - case '[Result "1-0"]': - blackWins = 0; - whiteWins = 1; - ties = 0; - whiteToBlackWinRatio = Infinity; - break; - case '[Result "1/2-1/2"]': - blackWins = 0.5; - whiteWins = 0.5; - ties = 1; - whiteToBlackWinRatio = 1; - break; - default: - blackWins = 0; - whiteWins = 0; - ties = 0; - whiteToBlackWinRatio = 0; + // add opening to openings object + if (this.openings[opening]) { + this.openings[opening].appearances++; + switch (result) { + case '[Result "1-0"]': + this.openings[opening].whiteWins++; + break; + case '[Result "0-1"]': + this.openings[opening].blackWins++; + break; + case '[Result "1/2-1/2"]': + this.openings[opening].whiteWins += 0.5; + this.openings[opening].blackWins += 0.5; + this.openings[opening].ties++; + break; + } + this.openings[opening].whiteToBlackWinRatio = + this.openings[opening].whiteWins / this.openings[opening].blackWins; + } else { + let blackWins, whiteWins, ties, whiteToBlackWinRatio; + + switch (result) { + case '[Result "0-1"]': + blackWins = 1; + whiteWins = 0; + ties = 0; + whiteToBlackWinRatio = 0; + break; + case '[Result "1-0"]': + blackWins = 0; + whiteWins = 1; + ties = 0; + whiteToBlackWinRatio = Infinity; + break; + case '[Result "1/2-1/2"]': + blackWins = 0.5; + whiteWins = 0.5; + ties = 1; + whiteToBlackWinRatio = 1; + break; + default: // Can this happen? Should it be an error? + blackWins = 0; + whiteWins = 0; + ties = 0; + whiteToBlackWinRatio = 0; + } + + this.openings[opening] = { + appearances: 1, + blackWins, + whiteWins, + ties, + whiteToBlackWinRatio, + }; + } } - this.openings[opening] = { - appearances: 1, - blackWins: blackWins, - whiteWins: whiteWins, - ties: ties, - whiteToBlackWinRatio: whiteToBlackWinRatio, - }; - } -} - - // identify game endings - let gameEnd = metadata.find(item => item.startsWith('[Termination')); + let gameEnd = metadata.find((item) => item.startsWith('[Termination')); const lastMove = game[game.length - 1].move; if (gameEnd === '[Termination "Normal"]') { if (lastMove.originalString.includes('#')) { - this.gameEndings['checkmate'] = (this.gameEndings['checkmate'] || 0) + 1; + this.gameEndings['checkmate'] = + (this.gameEndings['checkmate'] || 0) + 1; } else if (result === '[Result "1/2-1/2"]') { this.gameEndings['draw'] = (this.gameEndings['draw'] || 0) + 1; if (this.chess.isStalemate()) { - this.gameEndings['stalemate'] = (this.gameEndings['stalemate'] || 0) + 1; + this.gameEndings['stalemate'] = + (this.gameEndings['stalemate'] || 0) + 1; } else if (this.chess.isInsufficientMaterial()) { - this.gameEndings['insufficient material'] = (this.gameEndings['insufficient material'] || 0) + 1; - } + this.gameEndings['insufficient material'] = + (this.gameEndings['insufficient material'] || 0) + 1; + } } else { - this.gameEndings['resignation'] = (this.gameEndings['resignation'] || 0) + 1; + this.gameEndings['resignation'] = + (this.gameEndings['resignation'] || 0) + 1; } } else if (gameEnd === '[Termination "Time forfeit"]') { this.gameEndings['time out'] = (this.gameEndings['time out'] || 0) + 1; - } + } // check for threefold repetition (currently not checking if remaining castling rights and the possibility to capture en passant are the same per https://en.wikipedia.org/wiki/Threefold_repetition) // check for 50-moves rule let fiftyMovesCount = 0; let threefoldRepetitionCount = 0; let threefoldRepetitionFound = false; // check if threeFoldRepetition has been found - const lastBoardString = JSON.stringify(game[game.length - 1].board) + this.chess.turn(); + const lastBoardString = + JSON.stringify(game[game.length - 1].board) + this.chess.turn(); for (const { move, board } of game.reverse()) { const boardString = JSON.stringify(board) + this.chess.turn(); // check if the same player has the move if (boardString === lastBoardString) { threefoldRepetitionCount++; } - // only count as threefold repetition if the last board position is included in the repetition and + // only count as threefold repetition if the last board position is included in the repetition and // the last board position has appeared 3 times (including the last time) - if (!threefoldRepetitionFound && threefoldRepetitionCount === 3 && boardString === lastBoardString) { - this.gameEndings['threefold repetition'] = + if ( + !threefoldRepetitionFound && + threefoldRepetitionCount === 3 && + boardString === lastBoardString + ) { + this.gameEndings['threefold repetition'] = (this.gameEndings['threefold repetition'] || 0) + 1; threefoldRepetitionFound = true; } - + // check for fifty game rule if (move.capture || move.piece === 'p') { fiftyMovesCount = 0; } else { - fiftyMovesCount++ + fiftyMovesCount++; } if (fiftyMovesCount === 50) { - this.gameEndings['fifty-move rule'] = (this.gameEndings['fifty-move rule'] || 0) + 1; + this.gameEndings['fifty-move rule'] = + (this.gameEndings['fifty-move rule'] || 0) + 1; } } @@ -396,7 +411,9 @@ if(opening) { this.playerMostGames = playerMostGames; // sort gameTypeStats from greatest to least by number of times they appear - const sortedGameTypeStats = Object.entries(this.gameTypeStats).sort((a, b) => b[1] - a[1]); + const sortedGameTypeStats = Object.entries(this.gameTypeStats).sort( + (a, b) => b[1] - a[1] + ); const sortedGameTypeStatsObj = Object.fromEntries(sortedGameTypeStats); this.gameTypeStats = sortedGameTypeStatsObj as { numberUltraBulletGames: number; @@ -406,14 +423,20 @@ if(opening) { numberClassicalGames: number; numberOtherGames: number; }; - + // sort gameTimeControlStats from greatest to least by number of times they appear - const sortedGameTimeControlStats = Object.entries(this.gameTimeControlStats).sort((a, b) => b[1] - a[1]); - const sortedGameTimeControlStatsObj = Object.fromEntries(sortedGameTimeControlStats); + const sortedGameTimeControlStats = Object.entries( + this.gameTimeControlStats + ).sort((a, b) => b[1] - a[1]); + const sortedGameTimeControlStatsObj = Object.fromEntries( + sortedGameTimeControlStats + ); this.gameTimeControlStats = sortedGameTimeControlStatsObj; // sort the openings from greatest to least by number of times they appear - const sortedOpenings = Object.entries(this.openings).sort((a, b) => b[1].whiteToBlackWinRatio - a[1].whiteToBlackWinRatio); + const sortedOpenings = Object.entries(this.openings).sort( + (a, b) => b[1].whiteToBlackWinRatio - a[1].whiteToBlackWinRatio + ); const sortedOpeningsObj = Object.fromEntries(sortedOpenings); this.openings = sortedOpeningsObj; diff --git a/src/metrics/promotions.ts b/src/metrics/promotions.ts index db75cdc..c55d803 100644 --- a/src/metrics/promotions.ts +++ b/src/metrics/promotions.ts @@ -16,9 +16,10 @@ export class PromotionMetric implements Metric { }; // property for the totals totals: { [key in PromotablePiece]: number }; - movesAndGamesWithMaxQueenCount = []; - maxQueenCounts = 2; - + movesAndGamesWithMaxQueensOnBoard = []; + maxQueensOnBoard = 2; + maxQueens = 2; + movesAndGamesWithMaxQueens: { game: string; move: string }[] = []; constructor() { this.clear(); @@ -33,61 +34,94 @@ export class PromotionMetric implements Metric { } this.totals = { q: 0, r: 0, b: 0, n: 0 }; - this.promotionMap = promotionMap as any; - - this.movesAndGamesWithMaxQueenCount = []; + this.movesAndGamesWithMaxQueensOnBoard = []; + this.movesAndGamesWithMaxQueens = []; + this.maxQueensOnBoard = 2; + this.maxQueens = 2; } processGame( game: { move: PrettyMove; board: Piece[] }[], metadata?: string[] ) { - // 2 queens to start with - let thisGameQueenCount = 2; + let currentQueens = 2; + let totalQueens = 2; let gameSite = ''; if (metadata) { - gameSite = metadata.find((item) => item.startsWith('[Site "')) - ?.replace('[Site "', '') - ?.replace('"]', ''); + gameSite = metadata + .find((item) => item.startsWith('[Site "')) + ?.replace('[Site "', '') + ?.replace('"]', ''); } for (const { move } of game) { // TODO: we can use flags instead of includes('=) - if (move.originalString.includes('=')) { // update maps this.promotionMap[move.uas][move.promotion]++; - + // increment queen count if a promotion occurs - if (move.promotion === "q") { - thisGameQueenCount++; + if (move.promotion === 'q') { + currentQueens++; + totalQueens++; + // the check below is only necessary on promotions, when queen count might have increased beyond the max + + // identify the maxQueenCount in a particular move, and the games and moves that the maxQueenCount occured in + // push to array of games and moves if tie, otherwise wipe the array and add new game + // only add one move entry for each game maxQueenCount (rather than one entry for each move that the maxQueenCount appears in) + if (currentQueens > this.maxQueensOnBoard) { + this.maxQueensOnBoard = currentQueens; + this.movesAndGamesWithMaxQueensOnBoard = [ + { + game: gameSite, + move: move.originalString, + }, + ]; + } else if (currentQueens === this.maxQueensOnBoard) { + if ( + !this.movesAndGamesWithMaxQueensOnBoard.some( + (item) => item.game === gameSite + ) + ) { + this.movesAndGamesWithMaxQueensOnBoard.push({ + game: gameSite, + move: move.originalString, + }); + } + } } } // decrement queen count if queen is captured - if (move.capture && (move.capture.uas === "q" || move.capture.uas === "Q")) { - thisGameQueenCount--; + if ( + move.capture && + (move.capture.uas === 'q' || move.capture.uas === 'Q') + ) { + currentQueens--; } + } - // identify the maxQueenCount in a particular move, and the games and moves that the maxQueenCount occured in - // push to array of games and moves if tie, otherwise wipe the array and add new game - // only add one move entry for each game maxQueenCount (rather than one entry for each move that the maxQueenCount appears in) - if (thisGameQueenCount > this.maxQueenCounts) { - this.maxQueenCounts = thisGameQueenCount; - this.movesAndGamesWithMaxQueenCount = [{ + // after the game is done, update the maxQueens property and store the game it occured in + if (totalQueens > this.maxQueens) { + this.maxQueens = totalQueens; + this.movesAndGamesWithMaxQueens = [ + { game: gameSite, - move: move.originalString, - }]; - } else if (thisGameQueenCount === this.maxQueenCounts) { - if (!this.movesAndGamesWithMaxQueenCount.some((item) => item.game === gameSite)) { - this.movesAndGamesWithMaxQueenCount.push({ - game: gameSite, - move: move.originalString - }); - } + move: game[0].move.originalString, + }, + ]; + } else if (totalQueens === this.maxQueens) { + // this search could be slow. We shouldn't need this check + if ( + !this.movesAndGamesWithMaxQueens.some((item) => item.game === gameSite) + ) { + this.movesAndGamesWithMaxQueens.push({ + game: gameSite, + move: game[0].move.originalString, + }); } } } @@ -104,9 +138,9 @@ export class PromotionMetric implements Metric { return { promotedToTotals: this.totals, uasPromotingPieces: this.promotionMap, - maxNumQueens: this.maxQueenCounts, - movesAndGamesWithMaxQueenCount: this.movesAndGamesWithMaxQueenCount, - } + maxNumQueens: this.maxQueensOnBoard, + movesAndGamesWithMaxQueenCount: this.movesAndGamesWithMaxQueensOnBoard, + }; } logResults(): void { @@ -124,13 +158,17 @@ export class PromotionMetric implements Metric { console.log('\n'); // number of pieces to appear on board facts - console.log("NUMBER OF PIECES TO APPEAR ON BOARD FACTS:") - console.log(`The maximum number of queens to appear in a given move in a game: ${this.maxQueenCounts}`); + console.log('NUMBER OF PIECES TO APPEAR ON BOARD FACTS:'); + console.log( + `The maximum number of queens to appear in a given move in a game: ${this.maxQueensOnBoard}` + ); console.log(`The games(s) and first move(s) in that game in which that number of queens appeared: - ${this.movesAndGamesWithMaxQueenCount.map(move => - JSON.stringify(move, null, 2)).join(", ")}` + ${this.movesAndGamesWithMaxQueensOnBoard + .map((move) => JSON.stringify(move, null, 2)) + .join(', ')}`); + console.log( + '==============================================================' ); - console.log("=============================================================="); - console.log("\n"); + console.log('\n'); } } diff --git a/src/queue.ts b/src/queue.ts new file mode 100644 index 0000000..22dbbf7 --- /dev/null +++ b/src/queue.ts @@ -0,0 +1,44 @@ +import * as asyncLib from 'async'; +import * as fs from 'fs'; +import * as net from 'net'; +// will just write to wherever the process is running, but the server needs to be launched from the same directory so we use an abs path +export const RESULTS_PATH = `${__dirname}/results.json`; + +function launchQueueServer() { + // Create a write to result.json queue with a concurrency of 1 + // Possibly the simplest fix would be to run this as a separate process, then we can enforce messages sent to this queue are processed in order + const queue = asyncLib.queue((task, callback) => { + console.log('received task', task.analysisKey); + // return new Promise((resolve, reject) => { + const { results, analysisKey } = task; + try { + fs.writeFileSync(RESULTS_PATH, JSON.stringify(results, null, 2)); + console.log( + `Analysis "${analysisKey}" has been written to ${RESULTS_PATH}` + ); + } catch (err) { + console.error('Error writing to results.json', err); + } + // }); + }, 1); + + queue.drain(function () { + console.log('all items have been processed'); + }); + + // this event listener receives tasks from the parallel processes + const server = net.createServer((socket) => { + socket.on('data', (data) => { + const task = JSON.parse(data.toString()); + queue.push(task); + }); + }); + + console.log('Queue server listening on port 8000'); + server.listen(8000); +} + +// for use with zst_decompresser.js +if (require.main === module) { + launchQueueServer(); +} diff --git a/src/results.json b/src/results.json deleted file mode 100644 index 9e26dfe..0000000 --- a/src/results.json +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/src/run_metrics_on_file.ts b/src/run_metrics_on_file.ts new file mode 100644 index 0000000..c2f96bf --- /dev/null +++ b/src/run_metrics_on_file.ts @@ -0,0 +1,105 @@ +import * as fs from 'fs'; +import * as net from 'net'; +import { Chess } from '../cjsmin/src/chess'; +import { gameChunks } from './fileReader'; +import { + KDRatioMetric, + KillStreakMetric, + MateAndAssistMetric, +} from './metrics/captures'; +import { MoveDistanceMetric } from './metrics/distances'; +import { MetadataMetric } from './metrics/misc'; +import { + GameWithMostMovesMetric, + MiscMoveFactMetric, + PieceLevelMoveInfoMetric, +} from './metrics/moves'; +import { PromotionMetric } from './metrics/promotions'; +import { RESULTS_PATH } from './queue'; + +/** + * + * @param path + * @returns + */ +export async function main(path: string) { + console.time('Total Execution Time'); + await gameIterator(path, { 'Number of games analyzed': 0 }); + console.timeEnd('Total Execution Time'); + + const now = new Date(); + const milliseconds = now.getMilliseconds(); + + const analysisKey = `analysis_${now + .toLocaleString() + .replace(/\/|,|:|\s/g, '_')}_${milliseconds}`; + + let existingResults = {}; + if (fs.existsSync(RESULTS_PATH)) { + const fileContent = fs.readFileSync(RESULTS_PATH, 'utf8'); + if (fileContent !== '') { + existingResults = JSON.parse(fileContent); + } + } + + console.log('sending results'); + + // TODO: Probably we need to read in the existing results in the queue server and merge them, as when there are multiple items in the queue + // this is going to be out of date + existingResults[analysisKey] = { + 'Number of games analyzed': 0, + }; + + const client = net.createConnection({ port: 8000 }); + + console.log('connected to queue server'); + + // Send the task to the queue server + client.write(JSON.stringify({ results: existingResults, analysisKey })); + + console.log('results sent'); +} + +/** + * Metric functions will ingest a single game at a time + * @param metricFunctions + */ +async function gameIterator(path, results) { + const cjsmin = new Chess(); + + const gamesGenerator = gameChunks(path); + + const metrics = [ + new KDRatioMetric(), + new KillStreakMetric(), + new MateAndAssistMetric(), + new PromotionMetric(), + new MoveDistanceMetric(), + new GameWithMostMovesMetric(), + new PieceLevelMoveInfoMetric(), + new MetadataMetric(cjsmin), + new MiscMoveFactMetric(), + ]; + + let gameCounter = 0; + for await (const { moves, metadata } of gamesGenerator) { + if (gameCounter++ % 400 == 0) console.log(`ingested ${gameCounter} games`); + + for (const metric of metrics) { + // with array creation + const historyGenerator = cjsmin.historyGeneratorArr(moves); + // the generator is useless if we convert it to an array + metric.processGame(Array.from(historyGenerator), metadata); + } + } + + results['Number of games analyzed'] = gameCounter; + for (const metric of metrics) { + results[metric.constructor.name] = metric.aggregate(); + } +} + +// for use with zst_decompresser.js +if (require.main === module) { + main(process.argv[2]).then((results) => {}); +} diff --git a/src/scratch.ts b/src/scratch.ts deleted file mode 100644 index c5792b9..0000000 --- a/src/scratch.ts +++ /dev/null @@ -1,18 +0,0 @@ -// import { getPiecePromotionInfo } from '../src/metrics/metrics'; - -// // game being tested: https://lichess.org/jo73x9y8 -// const game = [ -// { -// metadata: [], -// moves: -// '1. e4 e5 2. Nf3 Nc6 3. d4 exd4 4. c3 Bc5 5. cxd4 Nxd4 6. Nxd4 Bxd4 7. Qxd4 d6 8. Qxg7 Qf6 9. Qxf6 Nxf6 10. Bc4 Be6 11. Bxe6 fxe6 12. Nc3 O-O-O 13. O-O Kb8 14. Bg5 Rhf8 15. Rad1 Rd7 16. e5 dxe5 17. Bxf6 Rxd1 18. Rxd1 a6 19. Bxe5 b5 20. Rd7 b4 21. Na4 Kb7 22. Rxc7+ Kb8 23. Rxh7+ Kc8 24. Rc7+ Kd8 25. Rc6 Ke7 26. Rxa6 Rd8 27. Ra7+ Ke8 28. Kf1 Rd2 29. Ra8+ Kd7 30. Rb8 Rd5 31. Bg3 Ra5 32. b3 Rd5 33. Rb7+ Kc6 34. Rb6+ Kd7 35. Rd6+ Rxd6 36. Bxd6 Kxd6 37. Ke2 Kc6 38. Kd3 Kb5 39. Ke4 Kc6 40. h4 Kd7 41. g4 Ke7 42. h5 Kf7 43. Ke5 Kg7 44. f4 Kh6 45. Kxe6 Kg7 46. g5 Kh7 47. Kf6 Kh8 48. g6 Kg8 49. h6 Kh8 50. Kg5 Kg8 51. f5 Kh8 52. f6 Kg8 53. g7 Kh7 54. Kf5 Kg8 55. Ke6 Kh7 56. f7 Kg6 57. f8=Q Kh7 58. g8=Q# 1-0', -// }, -// ]; - -// getPiecePromotionInfo(game) -// .then((result) => { -// console.log(result); -// }) -// .catch((error) => { -// console.error(error); -// }); diff --git a/src/streaming_partial_decompresser.js b/src/streaming_partial_decompresser.js deleted file mode 100644 index 2587158..0000000 --- a/src/streaming_partial_decompresser.js +++ /dev/null @@ -1,224 +0,0 @@ -const fs = require('fs'); -const zstd = require('node-zstandard'); -const { spawn } = require('child_process'); - -// List of all the database files you want to analyze (these need to be downloaded and in data folder) -const files = ["lichess_db_standard_rated_2018-05.pgn.zst", /*...*/]; - -// 30 games = 10*1024 bytes, 1 game = 350 bytes, 1000 games = 330KB, 100K games = 33MB -// 10MB yields around 30k games, 5GB = around 15 million games -const SIZE_LIMIT = 30 * 1024 * 1024 // 30MB - -// set the total size limit of the combined decompressed files (this is how much space you need to have available on your PC prior to running node src/streaming_partial_decompresser.js) -const decompressedSizeLimit = 500 * 1024 * 1024 * 1024 // 500 GB represented in bytes - -// function to check file size -const getFileSize = (filePath) => { - if (!fs.existsSync(filePath)) { - return 0; - } - const stats = fs.statSync(filePath); - return stats.size; -}; - -/** - * Runs the analysis script on a given file path. - * - * @param {string} filePath - The path of the file to run the analysis on. - * @return {Promise} A promise that resolves when the analysis is complete. - */ -const runAnalysis = (filePath) => { - return new Promise((resolve, reject) => { - // Run the analysis script - console.log(`Running analysis script on ${filePath}...`); - - const child = spawn('ts-node', ['/Users/bennyrubanov/Coding_Projects/chessanalysis/src/index_with_decompressor.ts', filePath]); - - // only log complete lines of output (no insertion of "stdout: " in the middle of a line) - // do this by accumulating the data until a newline character, and then logging the accumulated data - let accumulatedData = ''; - child.stdout.on('data', (data) => { - accumulatedData += data; - let newlineIndex; - while ((newlineIndex = accumulatedData.indexOf('\n')) >= 0) { - console.log(`stdout: ${accumulatedData.slice(0, newlineIndex)}`); - accumulatedData = accumulatedData.slice(newlineIndex + 1); - } - }); - - child.stderr.on('data', (data) => { - console.log(`stderr: ${data}`); - }); - - child.on('error', (error) => { - console.log(`error: ${error.message}`); - reject(error); - }); - - child.on('close', (code) => { - console.log(`child process exited with code ${code}`); - resolve(); - }); - }); -}; - -/** - * Decompresses and analyzes a file. - * - * @param {string} file - The name of the file to decompress and analyze. - * @param {number} [start=0] - The starting index for decompression. - * @returns {Promise} A promise that resolves when the decompression and analysis is complete. - */ -const decompressAndAnalyze = async (file, start = 0) => { - - let stopDecompression = false; - let these_chunks_counter = 0; // Initialize the chunk counter - let file_counter = 1; // Initialize the file counter - let total_chunk_counter = 0; - - const base_path = `/Users/bennyrubanov/Coding_Projects/chessanalysis/data/${file.replace('.zst', '')}`; - - // Create a new file path - let newFilePath = `${base_path}_${file_counter}`; - - // Create a new writable strxeam - console.log(`Creating file number ${file_counter}`); - let decompressedStream = fs.createWriteStream(newFilePath, { flags: 'a' }); - - // Check if file already exists - if (fs.existsSync(newFilePath)) { - const stats = fs.statSync(newFilePath); - start = stats.size; - } - - try { - await new Promise((resolve, reject) => { - console.log(`Starting decompression of chunk number ${total_chunk_counter}.`); - - let startTime = Date.now(); - - // https://www.npmjs.com/package/node-zstandard#decompressionstreamfromfile-inputfile-callback - zstd.decompressionStreamFromFile(`/Users/bennyrubanov/Coding_Projects/chessanalysis/data/${file}`, (err, result) => { - if (err) return reject(err); - - let lastChunkLength = 0; - let fileLength = 0; - let all_files_lengths = 0; - let batch_files_total_decompressed_size = 0; - let analysisPromises = []; - const MAX_CONCURRENT_ANALYSES = 13; - let filesBeingAnalyzed = new Set(); - - result.on('error', (err) => { - return reject(err); - }); - - result.on('data', async (data) => { - if (stopDecompression) { - return; // Skip writing and updating counters if stopDecompression is true - } - - decompressedStream.write(data); - lastChunkLength = data.length; - - const duration = Date.now() - startTime; - const durationFormatted = formatDuration(duration); - fileLength += data.length; - all_files_lengths += data.length; - batch_files_total_decompressed_size += data.length; - - // Increment the chunk counter - total_chunk_counter++; - these_chunks_counter++; - - if (total_chunk_counter % 200 === 0) { - console.log(`${these_chunks_counter} chunks decompressed with decompressed size ${fileLength / 1024 / 1024} MB`); - } - - // Check if the file size exceeds the limit - if (getFileSize(newFilePath) >= SIZE_LIMIT) { - console.log(`Finished decompression of data starting from byte ${start} and ending on byte ${start + fileLength} of ${file} in ${durationFormatted}`); - console.log(`Total number of chunks decompressed so far: ${total_chunk_counter}`); - console.log(`Total decompressed size of files decompressed ${all_files_lengths / 1024 / 1024} MB`); - - // Save the old path for analysis - let oldPath = newFilePath; - - // Increment the file counter - file_counter++; - - // Create a new file path - newFilePath = `${base_path}_${file_counter}`; - - // Stop decompression if the size of the combined decompressed files exceeds the decompressed total combined files size limit - if (batch_files_total_decompressed_size >= decompressedSizeLimit) { - console.log(`Decompressed size limit met. Ending decompression and finalizing analyses...`); - console.log(`Temp files being analyzed: ${filesBeingAnalyzed}`) - stopDecompression = true; // Set the flag to true to stop decompression - resolve(); // Resolve the promise to allow the 'end' event to handle the analysis - } - - // Switch to a new file - console.log(`Creating file number ${file_counter}`); - decompressedStream = fs.createWriteStream(newFilePath, { flags: 'a' }); - - // Add the old file to the set for analysis - filesBeingAnalyzed.add(oldPath); - - start += fileLength; - fileLength = 0; - these_chunks_counter = 0; - } - }); - - result.on('end', async () => { - // When all data is decompressed, run the analysis on the last file - let lastAnalysisPromise = runAnalysis(newFilePath).then(() => { - if (fs.existsSync(newFilePath)) { - fs.unlinkSync(newFilePath); - console.log(`File ${newFilePath} has been deleted.`); - } - }).catch(console.error); - - analysisPromises.push(lastAnalysisPromise); - filesBeingAnalyzed.add(newFilePath); - - // When all analyses are done, delete the files - Promise.allSettled(analysisPromises).then(() => { - console.log("All analyses completed"); - filesBeingAnalyzed.clear(); - }).catch(console.error); - - resolve(); - }); - }); - }); - - } catch (error) { - console.error(`Error decompressing data: ${error.message}`); - } -}; - -// Function to process all files -const processFiles = async () => { - console.log(`Initiating decompression and analysis of ${files}...`); - console.time('Final Total Compressed File Analysis Execution Time'); - for (const file of files) { - await decompressAndAnalyze(file); - } - console.timeEnd('Final Total Compressed File Analysis Execution Time'); -}; - -// Start the process -processFiles(); - -const formatDuration = (duration) => { - const hours = Math.floor(duration / 3600000); - const minutes = Math.floor((duration % 3600000) / 60000); - const seconds = Math.floor((duration % 60000) / 1000); - const milliseconds = duration % 1000; - - return `${hours}h ${minutes}m ${seconds}s ${milliseconds}ms`; -} - -module.exports = processFiles; \ No newline at end of file diff --git a/src/zst_decompressor.ts b/src/zst_decompressor.ts new file mode 100644 index 0000000..644ebac --- /dev/null +++ b/src/zst_decompressor.ts @@ -0,0 +1,308 @@ +import { randomUUID } from 'crypto'; +import * as path from 'path'; +import * as readline from 'readline'; + +// TODO: This should use type checking +const fs = require('fs'); +const zstd = require('node-zstandard'); +const { spawn } = require('child_process'); + +// 30 games = 10*1024 bytes, 1 game = 350 bytes, 1000 games = 330KB, 100K games = 33MB +// 10MB yields around 30k games, 5GB = around 15 million games +// const SIZE_LIMIT = 30 * 1024 * 1024; // 30MB +let SIZE_LIMIT = 10 * 1024 * 1024; // Default 10MB + +const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout +}); + +// Function to prompt for SIZE_LIMIT +const promptForSizeLimit = () => { + return new Promise((resolve) => { + rl.question('Enter the SIZE_LIMIT in MB (default is 10MB): ', (input) => { + const inputSizeMB = parseInt(input, 10); + if (!isNaN(inputSizeMB) && inputSizeMB > 0) { + SIZE_LIMIT = inputSizeMB * 1024 * 1024; // Convert MB to bytes + console.log(`Using SIZE_LIMIT of ${SIZE_LIMIT} bytes.`); + } else { + console.log(`Invalid input. Using default SIZE_LIMIT of ${SIZE_LIMIT} bytes.`); + } + resolve(); + }); + }); +}; + +let concurrentFilesLimit = 10; // How many files are analyzed at one time (batch size) + +// Function to prompt for concurrent files limit +const promptForConcurrentFilesLimit = () => { + return new Promise((resolve) => { + rl.question('Enter the number of files to analyze concurrently (default is 10): ', (input) => { + const inputLimit = parseInt(input, 10); + if (!isNaN(inputLimit) && inputLimit > 0) { + concurrentFilesLimit = inputLimit; + console.log(`Using concurrent files limit of ${concurrentFilesLimit}.`); + } else { + console.log(`Invalid input. Using default concurrent files limit of ${concurrentFilesLimit}.`); + } + resolve(); + }); + }); +}; + +// set the total size limit of the combined decompressed files (this is how much space you need to have available on your PC prior to running node src/streaming_partial_decompresser.js) +const decompressedSizeLimit = 500 * 1024 * 1024 * 1024; // 500 GB represented in bytes + +const getFileSize = (filePath: string) => { + if (!fs.existsSync(filePath)) { + return 0; + } + const stats = fs.statSync(filePath); + return stats.size; +}; + +/** + * Runs the analysis script on a given file path. + * @param {string} filePath - The path of the file to run the analysis on. + * @return {Promise} A promise that resolves when the analysis is complete. + */ +async function runAnalysis(filePath: string) { + return new Promise((resolve, reject) => { + // Run the analysis script + console.log(`Running analysis script on ${filePath}...`); + + const analysisFileBasePath = path.resolve(__dirname, '..', '..', 'src'); + + const child = spawn('ts-node', [ + `${analysisFileBasePath}/run_metrics_on_file.ts`, + filePath, + ]); + + // only log complete lines of output (no insertion of "stdout: " in the middle of a line) + // do this by accumulating the data until a newline character, and then logging the accumulated data + let accumulatedData = ''; + child.stdout.on('data', (data) => { + accumulatedData += data; + + let newlineIndex; + + // this loop slices data while theere is a newline chanracter in the accumulated data + while ((newlineIndex = accumulatedData.indexOf('\n')) >= 0) { + console.log(`stdout: ${accumulatedData.slice(0, newlineIndex)}`); + accumulatedData = accumulatedData.slice(newlineIndex + 1); + } + }); + + child.stderr.on('data', (data) => { + console.log(`stderr: ${data}`); + }); + + child.on('error', (error) => { + console.log(`error: ${error.message}`); + reject(error); + }); + + child.on('close', (code) => { + console.log(`child process exited with code ${code}`); + resolve(); + }); + }); +} + +/** + * Decompresses and analyzes a file. + * + * @param {string} file - The name of the file to decompress and analyze. + * @param {number} [start=0] - The starting index for decompression. + * @returns {Promise} A promise that resolves when the decompression and analysis is complete. + */ +const decompressAndAnalyze = async (file, start = 0) => { + let these_chunks_counter = 0; // Initialize the chunk counter + let file_counter = 1; // Initialize the file counter + let total_chunk_counter = 0; + const filesProduced = new Set(); + + // const base_path = `/Users/bennyrubanov/Coding_Projects/chessanalysis/data/${file.replace( + // base_path used to enumerate where new files should go + const base_path = path.resolve( + __dirname, + '..', + '..', + 'data', + file.replace('.zst', '') + ); + // for use in decompressionStreamFromFile + const compressedFilePath = path.resolve(__dirname, '..', '..', 'data'); + + // Create a new file path + let newFilePath = `${base_path}_${randomUUID()}`; + filesProduced.add(newFilePath); + + // Create a new writable strxeam + console.log(`Creating file #${file_counter} at ${newFilePath}`); + let decompressedStream = fs.createWriteStream(newFilePath, { flags: 'a' }); + + // Check if file already exists + if (fs.existsSync(newFilePath)) { + const stats = fs.statSync(newFilePath); + start = stats.size; + } + + try { + await new Promise((resolve, reject) => { + console.log( + `Starting decompression of chunk number ${total_chunk_counter}.` + ); + + let startTime = Date.now(); + + // https://www.npmjs.com/package/node-zstandard#decompressionstreamfromfile-inputfile-callback + zstd.decompressionStreamFromFile( + `${compressedFilePath}/${file}`, + (err, result) => { + if (err) return reject(err); + console.log( + `Decompressing file located at ${compressedFilePath}/${file}` + ); + + let fileLength = 0; + let batch_files_total_decompressed_size = 0; + let analysisPromises = []; + let filesBeingAnalyzed = new Set(); + + result.on('error', (err) => { + return reject(err); + }); + + result.on('data', async (data) => { + decompressedStream.write(data); + + const duration = Date.now() - startTime; + const durationFormatted = formatDuration(duration); + fileLength += data.length; + batch_files_total_decompressed_size += data.length; + these_chunks_counter++; + + // Check if the file size exceeds the limit, if so we need to make a new file + if (getFileSize(newFilePath) >= SIZE_LIMIT) { + console.log( + `Finished decompression of data starting from byte ${start} and ending on byte ${ + start + fileLength + } of ${file} in ${durationFormatted}` + ); + console.log( + `Total number of chunks decompressed so far: ${total_chunk_counter}` + ); + + // Increment the file counter + file_counter++; + + // Create a new file path + newFilePath = `${base_path}_${randomUUID()}`; + filesProduced.add(newFilePath); + + // Switch to a new file + console.log(`Creating file number ${file_counter}`); + decompressedStream = fs.createWriteStream(newFilePath, { + flags: 'a', + }); + + start += fileLength; + fileLength = 0; + total_chunk_counter += these_chunks_counter; + these_chunks_counter = 0; + + console.log( + `${these_chunks_counter} chunks decompressed with decompressed size ${ + fileLength / 1024 / 1024 + } MB` + ); + } + + // Stop decompression if the size of the combined decompressed files exceeds the decompressed total combined files size limit + if (batch_files_total_decompressed_size >= decompressedSizeLimit) { + console.log(`Decompression limit met. Ending decompression...`); + console.log(`Temp files being analyzed: ${filesBeingAnalyzed}`); + result.removeAllListeners('data'); + result.removeAllListeners('error'); + result.end(); + resolve(); // Resolve the promise to allow the 'end' event to handle the analysis + } + }); + + result.on('end', async () => { + // When all data is decompressed, run the analysis on the produced files concurrently + for (const file of Array.from(filesProduced).slice(0, 10)) { + // TODO: this won't work out of the box for a large number of files as there is no max concurrency. But the sample only produces 4 decompressed files + // I'm slicing to test this with a smaller number of files + + analysisPromises.push(runAnalysis(file)); + filesBeingAnalyzed.add(newFilePath); + } + + // When all analyses are done, delete the files from the set + Promise.allSettled(analysisPromises) + .then(() => { + console.log('All analyses completed'); + for (const file of filesBeingAnalyzed) { + if (fs.existsSync(file)) { + fs.unlinkSync(file); + console.log(`File ${file} has been deleted.`); + } + } + filesBeingAnalyzed.clear(); + }) + .catch(console.error); + + resolve(); + }); + } + ); + }); + } catch (error) { + console.error(`Error decompressing data: ${error.message}`); + } +}; + +// Function to process all files +const processFiles = async (files: string[]) => { + console.log(`Initiating decompression and analysis of files: ${files}...`); + console.time('Final Total Compressed File Analysis Execution Time'); + for (const file of files) { + try { + // Check if the file exists before proceeding + const filePath = path.resolve(__dirname, '..', '..', 'data', file); + if (!fs.existsSync(filePath)) { + throw new Error(`File does not exist: ${filePath}`); + } + await decompressAndAnalyze(file); + } catch (error) { + console.error(`Error processing file ${file}: ${error.message}`); + // Optionally, continue with the next file or handle the error as needed + } + } + console.timeEnd('Final Total Compressed File Analysis Execution Time'); +}; + +const formatDuration = (duration) => { + const hours = Math.floor(duration / 3600000); + const minutes = Math.floor((duration % 3600000) / 60000); + const seconds = Math.floor((duration % 60000) / 1000); + const milliseconds = duration % 1000; + + return `${hours}h ${minutes}m ${seconds}s ${milliseconds}ms`; +}; + +module.exports = processFiles; + +// run if main +if (require.main === module) { + promptForSizeLimit().then(() => { + promptForConcurrentFilesLimit().then(() => { + rl.close(); // Close the readline interface after all prompts + const files = ['lichess_db_standard_rated_2013-01.pgn.zst' /*...*/]; + processFiles(files); + }); + }); +} \ No newline at end of file