From c1dfadeed26d9217f36b5672546c3a5b6c889e97 Mon Sep 17 00:00:00 2001 From: Russell Steadman Date: Mon, 9 Oct 2023 09:03:13 +0000 Subject: [PATCH 1/4] Work on adding tests --- packages/bot/package-lock.json | 1408 ++++++++++++++++++++++++- packages/bot/package.json | 2 + packages/bot/src/index.ts | 23 +- packages/bot/src/test/_server.util.ts | 51 + packages/bot/src/test/bot.spec.ts | 83 ++ 5 files changed, 1502 insertions(+), 65 deletions(-) create mode 100644 packages/bot/src/test/_server.util.ts diff --git a/packages/bot/package-lock.json b/packages/bot/package-lock.json index d1615da..4da39ca 100644 --- a/packages/bot/package-lock.json +++ b/packages/bot/package-lock.json @@ -19,6 +19,7 @@ "@ava/typescript": "^4.1.0", "@swc/cli": "^0.1.62", "@swc/core": "^1.3.92", + "@types/express": "^4.17.18", "@typescript-eslint/eslint-plugin": "^6.7.4", "@typescript-eslint/parser": "^6.7.4", "ava": "^5.3.1", @@ -26,6 +27,7 @@ "eslint": "^8.51.0", "eslint-plugin-ava": "^14.0.0", "eslint-plugin-prettier": "^5.0.0", + "express": "^4.18.2", "prettier": "^3.0.3", "typescript": "^5.2.2" }, @@ -646,6 +648,16 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "dev": true }, + "node_modules/@types/body-parser": { + "version": "1.19.3", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.3.tgz", + "integrity": "sha512-oyl4jvAfTGX9Bt6Or4H9ni1Z447/tQuxnZsytsCaExKlmJiU8sFgnIBRzJUpKwB5eWn9HuBYlUlVA74q/yN0eQ==", + "dev": true, + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, "node_modules/@types/cacheable-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", @@ -658,11 +670,50 @@ "@types/responselike": "*" } }, + "node_modules/@types/connect": { + "version": "3.4.36", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz", + "integrity": "sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "4.17.18", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.18.tgz", + "integrity": "sha512-Sxv8BSLLgsBYmcnGdGjjEjqET2U+AKAdCRODmMiq02FgjwuV75Ut85DRpvFjyw/Mk0vgUOliGRU0UUmuuZHByQ==", + "dev": true, + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "4.17.37", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.37.tgz", + "integrity": "sha512-ZohaCYTgGFcOP7u6aJOhY9uIZQgZ2vxC2yWoArY+FeDXlqeH66ZVBjgvg+RLVAS/DWNq4Ap9ZXu1+SUQiiWYMg==", + "dev": true, + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, "node_modules/@types/http-cache-semantics": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz", "integrity": "sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==" }, + "node_modules/@types/http-errors": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.2.tgz", + "integrity": "sha512-lPG6KlZs88gef6aD85z3HNkztpj7w2R7HmR3gygjfXCQmsLloWNARFkMuzKiiY8FGdh1XDpgBdrSf4aKDiA7Kg==", + "dev": true + }, "node_modules/@types/istanbul-lib-coverage": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", @@ -684,11 +735,29 @@ "@types/node": "*" } }, + "node_modules/@types/mime": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz", + "integrity": "sha512-Ys+/St+2VF4+xuY6+kDIXGxbNRO0mesVg0bbxEfB97Od1Vjpjx9KD1qxs64Gcb3CWPirk9Xe+PT4YiiHQ9T+eg==", + "dev": true + }, "node_modules/@types/node": { "version": "20.8.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.3.tgz", "integrity": "sha512-jxiZQFpb+NlH5kjW49vXxvxTjeeqlbsnTAdBTKpzEdPs9itay7MscYXz3Fo9VYFEsfQ6LJFitHad3faerLAjCw==" }, + "node_modules/@types/qs": { + "version": "6.9.8", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.8.tgz", + "integrity": "sha512-u95svzDlTysU5xecFNTgfFG5RUWu1A9P0VzgpcIiGZA9iraHOdSzcxMxQ55DyeRaGCSxQi7LxXDI4rzq/MYfdg==", + "dev": true + }, + "node_modules/@types/range-parser": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.5.tgz", + "integrity": "sha512-xrO9OoVPqFuYyR/loIHjnbvvyRZREYKLjxV4+dY6v3FQR3stQ9ZxIGkaclF7YhI9hfjpuTbu14hZEy94qKLtOA==", + "dev": true + }, "node_modules/@types/responselike": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", @@ -704,6 +773,27 @@ "integrity": "sha512-OxepLK9EuNEIPxWNME+C6WwbRAOOI2o2BaQEGzz5Lu2e4Z5eDnEo+/aVEDMIXywoJitJ7xWd641wrGLZdtwRyw==", "dev": true }, + "node_modules/@types/send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.2.tgz", + "integrity": "sha512-aAG6yRf6r0wQ29bkS+x97BIs64ZLxeE/ARwyS6wrldMm3C1MdKwCcnnEwMC1slI8wuxJOpiUH9MioC0A0i+GJw==", + "dev": true, + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.3.tgz", + "integrity": "sha512-yVRvFsEMrv7s0lGhzrggJjNOSmZCdgCjw9xWrPr/kNNLp6FaDfMC1KaYl3TSJ0c58bECwNBMoQrZJ8hA8E1eFg==", + "dev": true, + "dependencies": { + "@types/http-errors": "*", + "@types/mime": "*", + "@types/node": "*" + } + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "6.7.4", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.4.tgz", @@ -893,6 +983,19 @@ "url": "https://opencollective.com/typescript-eslint" } }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/acorn": { "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", @@ -1027,6 +1130,12 @@ "node": ">=0.10.0" } }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true + }, "node_modules/array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -1513,6 +1622,45 @@ "integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==", "dev": true }, + "node_modules/body-parser": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "dev": true, + "dependencies": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/body-parser/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/body-parser/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, "node_modules/bplist-parser": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", @@ -1562,6 +1710,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/c8": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/c8/-/c8-8.0.1.tgz", @@ -1638,6 +1795,19 @@ "node": ">=8" } }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -1939,25 +2109,14 @@ "node": ">= 0.6" } }, - "node_modules/content-disposition/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] + "engines": { + "node": ">= 0.6" + } }, "node_modules/convert-to-spaces": { "version": "2.0.1", @@ -1968,6 +2127,21 @@ "node": "^12.20.0 || ^14.13.1 || >=16.0.0" } }, + "node_modules/cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -2108,6 +2282,25 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, "node_modules/dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -2138,6 +2331,12 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "dev": true }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true + }, "node_modules/emittery": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-1.0.1.tgz", @@ -2156,6 +2355,15 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, + "node_modules/encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -2186,6 +2394,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -2457,6 +2671,15 @@ "node": ">=0.10.0" } }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/exclusion": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/exclusion/-/exclusion-0.1.1.tgz", @@ -2518,6 +2741,63 @@ "node": ">=0.10.0" } }, + "node_modules/express": { + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "dev": true, + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + } + }, + "node_modules/express/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, "node_modules/ext-list": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/ext-list/-/ext-list-2.2.2.tgz", @@ -2702,6 +2982,39 @@ "node": ">=8" } }, + "node_modules/finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/finalhandler/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/finalhandler/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -2773,6 +3086,24 @@ "node": ">= 14.17" } }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -2793,6 +3124,12 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -2802,6 +3139,21 @@ "node": "6.* || 8.* || >= 10.*" } }, + "node_modules/get-intrinsic": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", + "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", + "dev": true, + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -2974,6 +3326,15 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/has": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.4.tgz", + "integrity": "sha512-qdSAmqLF6209RFj4VVItywPMbm3vWylknmB3nvNiUIs72xAimcM8nVYxYr7ncvZq5qzk9MKIZR8ijqD/1QuYjQ==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -2983,6 +3344,30 @@ "node": ">=8" } }, + "node_modules/has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -2994,6 +3379,22 @@ "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==" }, + "node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/http2-wrapper": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.0.tgz", @@ -3015,6 +3416,18 @@ "node": ">=14.18.0" } }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -3118,6 +3531,15 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/irregular-plurals": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.3.0.tgz", @@ -3521,6 +3943,15 @@ "node": ">=8" } }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mem": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/mem/-/mem-9.0.2.tgz", @@ -3537,6 +3968,12 @@ "url": "https://github.com/sindresorhus/mem?sponsor=1" } }, + "node_modules/merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==", + "dev": true + }, "node_modules/merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -3552,6 +3989,15 @@ "node": ">= 8" } }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/micro-spelling-correcter": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/micro-spelling-correcter/-/micro-spelling-correcter-1.1.1.tgz", @@ -3571,6 +4017,18 @@ "node": ">=8.6" } }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true, + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -3580,6 +4038,18 @@ "node": ">= 0.6" } }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-fn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", @@ -3625,6 +4095,15 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/nofilter": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", @@ -3682,6 +4161,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -3875,6 +4375,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -3902,6 +4411,12 @@ "node": ">=8" } }, + "node_modules/path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", + "dev": true + }, "node_modules/path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -4106,6 +4621,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -4131,6 +4659,21 @@ "node": ">=6" } }, + "node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -4162,6 +4705,30 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dev": true, + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -4426,6 +4993,32 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, "node_modules/semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -4468,6 +5061,51 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dev": true, + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/send/node_modules/debug/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, "node_modules/serialize-error": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", @@ -4495,6 +5133,27 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dev": true, + "dependencies": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -4516,6 +5175,20 @@ "node": ">=8" } }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -4622,6 +5295,15 @@ "node": ">=8" } }, + "node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -4631,26 +5313,6 @@ "safe-buffer": "~5.2.0" } }, - "node_modules/string_decoder/node_modules/safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, "node_modules/string-width": { "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", @@ -4893,6 +5555,15 @@ "node": ">=8.0" } }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true, + "engines": { + "node": ">=0.6" + } + }, "node_modules/token-types": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz", @@ -4976,6 +5647,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/typescript": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", @@ -4989,6 +5673,15 @@ "node": ">=14.17" } }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -5013,6 +5706,15 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/v8-to-istanbul": { "version": "9.1.3", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.3.tgz", @@ -5033,6 +5735,15 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", "dev": true }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", @@ -5552,6 +6263,16 @@ "integrity": "sha512-OvjF+z51L3ov0OyAU0duzsYuvO01PH7x4t6DJx+guahgTnBHkhJdG7soQeTSFLWN3efnHyibZ4Z8l2EuWwJN3A==", "dev": true }, + "@types/body-parser": { + "version": "1.19.3", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.3.tgz", + "integrity": "sha512-oyl4jvAfTGX9Bt6Or4H9ni1Z447/tQuxnZsytsCaExKlmJiU8sFgnIBRzJUpKwB5eWn9HuBYlUlVA74q/yN0eQ==", + "dev": true, + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, "@types/cacheable-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", @@ -5564,11 +6285,50 @@ "@types/responselike": "*" } }, + "@types/connect": { + "version": "3.4.36", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.36.tgz", + "integrity": "sha512-P63Zd/JUGq+PdrM1lv0Wv5SBYeA2+CORvbrXbngriYY0jzLUWfQMQQxOhjONEz/wlHOAxOdY7CY65rgQdTjq2w==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/express": { + "version": "4.17.18", + "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.18.tgz", + "integrity": "sha512-Sxv8BSLLgsBYmcnGdGjjEjqET2U+AKAdCRODmMiq02FgjwuV75Ut85DRpvFjyw/Mk0vgUOliGRU0UUmuuZHByQ==", + "dev": true, + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^4.17.33", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "4.17.37", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.37.tgz", + "integrity": "sha512-ZohaCYTgGFcOP7u6aJOhY9uIZQgZ2vxC2yWoArY+FeDXlqeH66ZVBjgvg+RLVAS/DWNq4Ap9ZXu1+SUQiiWYMg==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, "@types/http-cache-semantics": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.2.tgz", "integrity": "sha512-FD+nQWA2zJjh4L9+pFXqWOi0Hs1ryBCfI+985NjluQ1p8EYtoLvjLOKidXBtZ4/IcxDX4o8/E8qDS3540tNliw==" }, + "@types/http-errors": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.2.tgz", + "integrity": "sha512-lPG6KlZs88gef6aD85z3HNkztpj7w2R7HmR3gygjfXCQmsLloWNARFkMuzKiiY8FGdh1XDpgBdrSf4aKDiA7Kg==", + "dev": true + }, "@types/istanbul-lib-coverage": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz", @@ -5590,11 +6350,29 @@ "@types/node": "*" } }, + "@types/mime": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.3.tgz", + "integrity": "sha512-Ys+/St+2VF4+xuY6+kDIXGxbNRO0mesVg0bbxEfB97Od1Vjpjx9KD1qxs64Gcb3CWPirk9Xe+PT4YiiHQ9T+eg==", + "dev": true + }, "@types/node": { "version": "20.8.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.3.tgz", "integrity": "sha512-jxiZQFpb+NlH5kjW49vXxvxTjeeqlbsnTAdBTKpzEdPs9itay7MscYXz3Fo9VYFEsfQ6LJFitHad3faerLAjCw==" }, + "@types/qs": { + "version": "6.9.8", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.8.tgz", + "integrity": "sha512-u95svzDlTysU5xecFNTgfFG5RUWu1A9P0VzgpcIiGZA9iraHOdSzcxMxQ55DyeRaGCSxQi7LxXDI4rzq/MYfdg==", + "dev": true + }, + "@types/range-parser": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.5.tgz", + "integrity": "sha512-xrO9OoVPqFuYyR/loIHjnbvvyRZREYKLjxV4+dY6v3FQR3stQ9ZxIGkaclF7YhI9hfjpuTbu14hZEy94qKLtOA==", + "dev": true + }, "@types/responselike": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz", @@ -5610,6 +6388,27 @@ "integrity": "sha512-OxepLK9EuNEIPxWNME+C6WwbRAOOI2o2BaQEGzz5Lu2e4Z5eDnEo+/aVEDMIXywoJitJ7xWd641wrGLZdtwRyw==", "dev": true }, + "@types/send": { + "version": "0.17.2", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.2.tgz", + "integrity": "sha512-aAG6yRf6r0wQ29bkS+x97BIs64ZLxeE/ARwyS6wrldMm3C1MdKwCcnnEwMC1slI8wuxJOpiUH9MioC0A0i+GJw==", + "dev": true, + "requires": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "@types/serve-static": { + "version": "1.15.3", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.3.tgz", + "integrity": "sha512-yVRvFsEMrv7s0lGhzrggJjNOSmZCdgCjw9xWrPr/kNNLp6FaDfMC1KaYl3TSJ0c58bECwNBMoQrZJ8hA8E1eFg==", + "dev": true, + "requires": { + "@types/http-errors": "*", + "@types/mime": "*", + "@types/node": "*" + } + }, "@typescript-eslint/eslint-plugin": { "version": "6.7.4", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.4.tgz", @@ -5710,6 +6509,16 @@ "eslint-visitor-keys": "^3.4.1" } }, + "accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "dev": true, + "requires": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + } + }, "acorn": { "version": "8.10.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.10.0.tgz", @@ -5794,6 +6603,12 @@ "integrity": "sha1-3wEKoSh+Fku9pvlyOwqWoexBh6E=", "dev": true }, + "array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "dev": true + }, "array-union": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", @@ -6140,6 +6955,43 @@ "integrity": "sha512-DRQrD6gJyy8FbiE4s+bDoXS9hiW3Vbx5uCdwvcCf3zLHL+Iv7LtGHLpr+GZV8rHG8tK766FGYBwRbu8pELTt+w==", "dev": true }, + "body-parser": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", + "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "dev": true, + "requires": { + "bytes": "3.1.2", + "content-type": "~1.0.4", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "on-finished": "2.4.1", + "qs": "6.11.0", + "raw-body": "2.5.1", + "type-is": "~1.6.18", + "unpipe": "1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + } + } + }, "bplist-parser": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/bplist-parser/-/bplist-parser-0.2.0.tgz", @@ -6177,6 +7029,12 @@ "run-applescript": "^5.0.0" } }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "dev": true + }, "c8": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/c8/-/c8-8.0.1.tgz", @@ -6234,6 +7092,16 @@ } } }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -6450,22 +7318,32 @@ "dev": true, "requires": { "safe-buffer": "5.2.1" - }, - "dependencies": { - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true - } } }, + "content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "dev": true + }, "convert-to-spaces": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz", "integrity": "sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ==", "dev": true }, + "cookie": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", + "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "dev": true + }, + "cookie-signature": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", + "integrity": "sha512-QADzlaHc8icV8I7vbaJXJwod9HWYp8uCqf1xa4OfNu1T7JVxQIrUgOWtHdNDtPiywmFbiS12VjotIXLrKM3orQ==", + "dev": true + }, "cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -6558,6 +7436,18 @@ "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", "dev": true }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "dev": true + }, + "destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "dev": true + }, "dir-glob": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", @@ -6582,6 +7472,12 @@ "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", "dev": true }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "dev": true + }, "emittery": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-1.0.1.tgz", @@ -6594,6 +7490,12 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, + "encodeurl": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", + "integrity": "sha512-TPJXq8JqFaVYm2CWmPvnP2Iyo4ZSM7/QKcSmuMLDObfpH5fi7RUGmd/rTDf+rut/saiDiQEeVTNgAmJEdAOx0w==", + "dev": true + }, "end-of-stream": { "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", @@ -6618,6 +7520,12 @@ "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", "dev": true }, + "escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "dev": true + }, "escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -6798,6 +7706,12 @@ "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", "dev": true }, + "etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "dev": true + }, "exclusion": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/exclusion/-/exclusion-0.1.1.tgz", @@ -6848,6 +7762,62 @@ } } }, + "express": { + "version": "4.18.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", + "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "dev": true, + "requires": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "1.20.1", + "content-disposition": "0.5.4", + "content-type": "~1.0.4", + "cookie": "0.5.0", + "cookie-signature": "1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "1.2.0", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "merge-descriptors": "1.0.1", + "methods": "~1.1.2", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "0.1.7", + "proxy-addr": "~2.0.7", + "qs": "6.11.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "0.18.0", + "serve-static": "1.15.0", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + } + } + }, "ext-list": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/ext-list/-/ext-list-2.2.2.tgz", @@ -6988,6 +7958,38 @@ "to-regex-range": "^5.0.1" } }, + "finalhandler": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.2.0.tgz", + "integrity": "sha512-5uXcUVftlQMFnWC9qu/svkWv3GTd2PfUhK/3PLkYNAe7FbqJMt3515HaxE6eRL74GdsriiwujiawdaB1BpEISg==", + "dev": true, + "requires": { + "debug": "2.6.9", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "on-finished": "2.4.1", + "parseurl": "~1.3.3", + "statuses": "2.0.1", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + } + } + }, "find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -7038,6 +8040,18 @@ "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-2.1.4.tgz", "integrity": "sha512-yDYSgNMraqvnxiEXO4hi88+YZxaHC6QKzb5N84iRCTDeRO7ZALpir/lVmf/uXUhnwUr2O4HU8s/n6x+yNjQkHw==" }, + "forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "dev": true + }, + "fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "dev": true + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -7051,12 +8065,30 @@ "dev": true, "optional": true }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", + "dev": true + }, "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", "dev": true }, + "get-intrinsic": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", + "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", + "dev": true, + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3" + } + }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -7171,12 +8203,30 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "has": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.4.tgz", + "integrity": "sha512-qdSAmqLF6209RFj4VVItywPMbm3vWylknmB3nvNiUIs72xAimcM8nVYxYr7ncvZq5qzk9MKIZR8ijqD/1QuYjQ==", + "dev": true + }, "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, + "has-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.1.tgz", + "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==", + "dev": true + }, + "has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true + }, "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -7188,6 +8238,19 @@ "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.1.tgz", "integrity": "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ==" }, + "http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dev": true, + "requires": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + } + }, "http2-wrapper": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/http2-wrapper/-/http2-wrapper-2.2.0.tgz", @@ -7203,6 +8266,15 @@ "integrity": "sha512-nZXjEF2nbo7lIw3mgYjItAfgQXog3OjJogSbKa2CQIIvSGWcKgeJnQlNXip6NglNzYH45nSRiEVimMvYL8DDqQ==", "dev": true }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dev": true, + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -7265,6 +8337,12 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", "dev": true }, + "ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "dev": true + }, "irregular-plurals": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.3.0.tgz", @@ -7543,6 +8621,12 @@ "blueimp-md5": "^2.10.0" } }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "dev": true + }, "mem": { "version": "9.0.2", "resolved": "https://registry.npmjs.org/mem/-/mem-9.0.2.tgz", @@ -7553,6 +8637,12 @@ "mimic-fn": "^4.0.0" } }, + "merge-descriptors": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", + "integrity": "sha512-cCi6g3/Zr1iqQi6ySbseM1Xvooa98N0w31jzUYrXPX2xqObmFGHJ0tQ5u74H3mVh7wLouTseZyYIq39g8cNp1w==", + "dev": true + }, "merge-stream": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", @@ -7565,6 +8655,12 @@ "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", "dev": true }, + "methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "dev": true + }, "micro-spelling-correcter": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/micro-spelling-correcter/-/micro-spelling-correcter-1.1.1.tgz", @@ -7581,12 +8677,27 @@ "picomatch": "^2.2.3" } }, + "mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "dev": true + }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", "dev": true }, + "mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dev": true, + "requires": { + "mime-db": "1.52.0" + } + }, "mimic-fn": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-4.0.0.tgz", @@ -7620,6 +8731,12 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "dev": true + }, "nofilter": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/nofilter/-/nofilter-3.1.0.tgz", @@ -7655,6 +8772,21 @@ } } }, + "object-inspect": { + "version": "1.12.3", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz", + "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g==", + "dev": true + }, + "on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "dev": true, + "requires": { + "ee-first": "1.1.1" + } + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", @@ -7782,6 +8914,12 @@ "integrity": "sha512-Tpb8Z7r7XbbtBTrM9UhpkzzaMrqA2VXMT3YChzYltwV3P3pM6t8wl7TvpMnSTosz1aQAdVib7kdoys7vYOPerw==", "dev": true }, + "parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "dev": true + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -7800,6 +8938,12 @@ "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "dev": true }, + "path-to-regexp": { + "version": "0.1.7", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", + "integrity": "sha512-5DFkuoqlv1uYQKxy8omFBeJPQcdoE07Kv2sferDCrAq1ohOU+MSDswDIbnx3YAM60qIOnYa53wBhXW0EbMonrQ==", + "dev": true + }, "path-type": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", @@ -7924,6 +9068,16 @@ "parse-ms": "^3.0.0" } }, + "proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "dev": true, + "requires": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + } + }, "pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -7946,6 +9100,15 @@ "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", "dev": true }, + "qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dev": true, + "requires": { + "side-channel": "^1.0.4" + } + }, "queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -7957,6 +9120,24 @@ "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==" }, + "range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "dev": true + }, + "raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dev": true, + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "readable-stream": { "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", @@ -8136,6 +9317,18 @@ "queue-microtask": "^1.2.2" } }, + "safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "dev": true + }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "dev": true + }, "semver": { "version": "7.5.4", "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", @@ -8160,6 +9353,52 @@ "semver": "^7.3.5" } }, + "send": { + "version": "0.18.0", + "resolved": "https://registry.npmjs.org/send/-/send-0.18.0.tgz", + "integrity": "sha512-qqWzuOjSFOuqPjFe4NOsMLafToQQwBSOEpS+FwEt3A2V3vKubTquT3vmLTQpFgMXp8AlFWFuP1qKaJZOtPpVXg==", + "dev": true, + "requires": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "0.5.2", + "http-errors": "2.0.0", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "2.4.1", + "range-parser": "~1.2.1", + "statuses": "2.0.1" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dev": true, + "requires": { + "ms": "2.0.0" + }, + "dependencies": { + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "dev": true + } + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + } + } + }, "serialize-error": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", @@ -8177,6 +9416,24 @@ } } }, + "serve-static": { + "version": "1.15.0", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.15.0.tgz", + "integrity": "sha512-XGuRDNjXUijsUL0vl6nSD7cwURuzEgglbOaFuZM9g3kwDXOWVTck0jLzjPzGD+TazWbboZYu52/9/XPdUgne9g==", + "dev": true, + "requires": { + "encodeurl": "~1.0.2", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "0.18.0" + } + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "dev": true + }, "shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -8192,6 +9449,17 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dev": true, + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", @@ -8269,6 +9537,12 @@ } } }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "dev": true + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -8276,14 +9550,6 @@ "dev": true, "requires": { "safe-buffer": "~5.2.0" - }, - "dependencies": { - "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", - "dev": true - } } }, "string-width": { @@ -8451,6 +9717,12 @@ "is-number": "^7.0.0" } }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "dev": true + }, "token-types": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/token-types/-/token-types-5.0.1.tgz", @@ -8506,12 +9778,28 @@ "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", "dev": true }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "dev": true, + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, "typescript": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "dev": true }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "dev": true + }, "untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -8533,6 +9821,12 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "dev": true + }, "v8-to-istanbul": { "version": "9.1.3", "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.1.3.tgz", @@ -8552,6 +9846,12 @@ } } }, + "vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "dev": true + }, "well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", diff --git a/packages/bot/package.json b/packages/bot/package.json index caa3e1c..78d6106 100644 --- a/packages/bot/package.json +++ b/packages/bot/package.json @@ -47,6 +47,7 @@ "@ava/typescript": "^4.1.0", "@swc/cli": "^0.1.62", "@swc/core": "^1.3.92", + "@types/express": "^4.17.18", "@typescript-eslint/eslint-plugin": "^6.7.4", "@typescript-eslint/parser": "^6.7.4", "ava": "^5.3.1", @@ -54,6 +55,7 @@ "eslint": "^8.51.0", "eslint-plugin-ava": "^14.0.0", "eslint-plugin-prettier": "^5.0.0", + "express": "^4.18.2", "prettier": "^3.0.3", "typescript": "^5.2.2" }, diff --git a/packages/bot/src/index.ts b/packages/bot/src/index.ts index 833d2e8..2982fcb 100644 --- a/packages/bot/src/index.ts +++ b/packages/bot/src/index.ts @@ -2,7 +2,7 @@ import QuickLRU from 'quick-lru'; import CacheableLookup from 'cacheable-lookup'; import * as APIError from './errors.js'; import { RobotsTxt } from 'exclusion'; -import got, { type Request, type Response } from 'got'; +import got, { Options, type Request, type Response } from 'got'; type BotOptions = { name: string; @@ -21,13 +21,13 @@ type BotRequestOptions = Partial<{ class Bot { private robotsTxt = {} as Record; - private robotsTxtString = {} as Record; + private robotsTxtExpires = {} as Record; readonly userAgent!: string; readonly botName!: string; private requestDelay = {} as Record; private requestTime = {} as Record; - private cache!: QuickLRU; - private dnsCachable!: CacheableLookup; + cache!: QuickLRU; + dnsCachable!: CacheableLookup; private options!: BotOptions; constructor(options: BotOptions) { @@ -103,7 +103,7 @@ class Bot { } } - const defaultOptions = { + const defaultOptions: Partial = { timeout: { lookup: 6e4, socket: 6e4, @@ -133,6 +133,7 @@ class Bot { cache: this.cache, cacheOptions: { shared: false, + immutableMinTimeToLive: 3600 * 1000, }, dnsCache: this.dnsCachable, }; @@ -144,7 +145,7 @@ class Bot { return got.get(rawURL, { ...defaultOptions, responseType: 'text', - }); + }) as Request | Response; } private async fetchRobotsTxt(origin: string) { @@ -153,10 +154,10 @@ class Bot { try { const robotsTxt = await this.fetchURL(robotsTxtURL); - if (this.robotsTxtString[origin] !== robotsTxt.body) { - this.robotsTxtString[origin] = robotsTxt.body; - this.robotsTxt[origin] = new RobotsTxt(robotsTxt.body); - } + this.robotsTxtExpires[origin] = new Date( + new Date().getTime() + 24 * 60 * 60 * 1000, + ); + this.robotsTxt[origin] = new RobotsTxt(robotsTxt.body); } catch (err) { console.error(err); } @@ -176,7 +177,7 @@ class Bot { await this.fetchRobotsTxt(url.origin); - const allowedByRobots = this.robotsTxt[origin].isPathAllowed( + const allowedByRobots = this.robotsTxt[url.origin].isPathAllowed( `${url.pathname}${url.search}`, this.botName, ); diff --git a/packages/bot/src/test/_server.util.ts b/packages/bot/src/test/_server.util.ts new file mode 100644 index 0000000..06fd67a --- /dev/null +++ b/packages/bot/src/test/_server.util.ts @@ -0,0 +1,51 @@ +import express, { type Request } from 'express'; +import { type Server } from 'http'; + +let currentPort = 3000; +const getPort = () => currentPort++; + +export function createServer(robotsTxt: string) { + const app = express(); + const requests: Request[] = []; + app.get('/', (req, res) => { + requests.push(req); + res.send('Home'); + }); + app.get('/a', (req, res) => { + requests.push(req); + res.send('A'); + }); + app.get('/a/b', (req, res) => { + requests.push(req); + res.send('B'); + }); + app.get('/a/b/c', (req, res) => { + requests.push(req); + res.send('C'); + }); + app.get('/robots.txt', (req, res) => { + requests.push(req); + res.send(robotsTxt); + }); + return { app, requests }; +} + +export async function startServer(robotsTxt: string) { + const { app, requests } = createServer(robotsTxt); + const port = getPort(); + const server = await new Promise((res) => { + const s = app.listen(port, () => { + res(s); + }); + }); + + return { server, requests, port }; +} + +export async function stopServer(server: Server) { + await new Promise((res) => { + server.close(() => { + res(undefined); + }); + }); +} diff --git a/packages/bot/src/test/bot.spec.ts b/packages/bot/src/test/bot.spec.ts index ba1e01c..c89dce4 100644 --- a/packages/bot/src/test/bot.spec.ts +++ b/packages/bot/src/test/bot.spec.ts @@ -1,7 +1,90 @@ import test from 'ava'; import Bot from '../index.js'; +import { type Server } from 'http'; +import { startServer, stopServer } from './_server.util.js'; + +let server: Server | undefined; test('Bot#constructor()', (it) => { const bot = new Bot({ name: 'Test', version: '0.1' }); it.is(typeof bot.makeRequest, 'function'); }); + +test('Bot disallow none', async (it) => { + const start = await startServer('User-agent: *\nDisallow:'); + server = start.server; + + const bot = new Bot({ name: 'Test', version: '0.1' }); + + let req = await bot.makeRequest(`http://127.0.0.1:${start.port}/`); + it.is(req.body, 'Home'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a`); + it.is(req.body, 'A'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`); + it.is(req.body, 'B'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`); + it.is(req.body, 'C'); + + const [req1, req2, req3, req4, req5] = start.requests; + + console.log('Cache size', bot.cache.size); + + console.log(req1); + + it.is(req1.path, '/robots.txt'); + it.is(req2.path, '/'); + it.is(req3.path, '/a'); + it.is(req4.path, '/a/b'); + it.is(req5.path, '/a/b/c'); +}); + +test('Bot allow all', async (it) => { + const start = await startServer('User-agent: *\nAllow: /'); + server = start.server; + + const bot = new Bot({ name: 'Test', version: '0.1' }); + + let req = await bot.makeRequest(`http://127.0.0.1:${start.port}/`); + it.is(req.body, 'Home'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a`); + it.is(req.body, 'A'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`); + it.is(req.body, 'B'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`); + it.is(req.body, 'C'); +}); + +test('Bot disallow all', async (it) => { + const start = await startServer('User-agent: *\nDisallow: /'); + server = start.server; + + const bot = new Bot({ name: 'Test', version: '0.1' }); + + await it.throwsAsync(() => + bot.makeRequest(`http://127.0.0.1:${start.port}/`), + ); + + await it.throwsAsync(() => + bot.makeRequest(`http://127.0.0.1:${start.port}/a`), + ); + + await it.throwsAsync(() => + bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`), + ); + + await it.throwsAsync(() => + bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`), + ); +}); + +test.afterEach(async () => { + if (server) { + await stopServer(server); + } +}); From 851e64eabace73331543ec24c1f2a36405bfb9e2 Mon Sep 17 00:00:00 2001 From: Russell Steadman Date: Wed, 11 Oct 2023 23:10:34 +0000 Subject: [PATCH 2/4] Add comments and error handling --- packages/bot/src/errors.ts | 28 ++--- packages/bot/src/index.ts | 158 +++++++++++++++++++------- packages/bot/src/test/_server.util.ts | 18 ++- packages/bot/src/test/bot.spec.ts | 110 ++++++++++++++++-- 4 files changed, 242 insertions(+), 72 deletions(-) diff --git a/packages/bot/src/errors.ts b/packages/bot/src/errors.ts index 32c90d1..b135e86 100644 --- a/packages/bot/src/errors.ts +++ b/packages/bot/src/errors.ts @@ -1,23 +1,23 @@ -export class BadRequest extends Error implements Error { - status = 400; +enum BotErrorType { + Request, + Delay, + RobotsTxt, + Configuration, } -export class NotAuthorized extends Error implements Error { - status = 401; +export class BotError extends Error implements Error { + status?: number; + type: BotErrorType = BotErrorType.Request; } -export class NotFound extends Error implements Error { - status = 404; +export class RobotsRejection extends BotError implements BotError { + type = BotErrorType.RobotsTxt; } -export class Unprocessable extends Error implements Error { - status = 422; +export class DelayError extends BotError implements BotError { + type = BotErrorType.Delay; } -export class Internal extends Error implements Error { - status = 500; -} - -export class BadGateway extends Error implements Error { - status = 502; +export class ConfigError extends BotError implements BotError { + type = BotErrorType.Configuration; } diff --git a/packages/bot/src/index.ts b/packages/bot/src/index.ts index 2982fcb..21a53bd 100644 --- a/packages/bot/src/index.ts +++ b/packages/bot/src/index.ts @@ -1,6 +1,6 @@ import QuickLRU from 'quick-lru'; import CacheableLookup from 'cacheable-lookup'; -import * as APIError from './errors.js'; +import * as Errors from './errors.js'; import { RobotsTxt } from 'exclusion'; import got, { Options, type Request, type Response } from 'got'; @@ -8,6 +8,7 @@ type BotOptions = { name: string; version: string; minimumRequestDelay?: number; + maximumRequestDelay?: number; disableCaching?: boolean; policyURL?: string; hideLibraryAgent?: boolean; @@ -17,11 +18,11 @@ type BotOptions = { type BotRequestOptions = Partial<{ stream: boolean; headers: Record; + overrides: Partial; }>; class Bot { private robotsTxt = {} as Record; - private robotsTxtExpires = {} as Record; readonly userAgent!: string; readonly botName!: string; private requestDelay = {} as Record; @@ -31,45 +32,83 @@ class Bot { private options!: BotOptions; constructor(options: BotOptions) { + // Validate the options if (!options || typeof options !== 'object') { - throw new Error('Missing or misformatted Bot options'); + throw new Errors.ConfigError('Missing or misformatted Bot options'); } + // Validate the bot name if (!options.name || typeof options.name !== 'string') { - throw new Error('Bot name must be a string'); + throw new Errors.ConfigError('Bot name must be a string'); } else if (!/^[a-zA-Z_-]+$/.test(options.name)) { - throw new Error('Bot name must only contain a-zA-Z_-'); + throw new Errors.ConfigError('Bot name must only contain a-zA-Z_-'); } + // Validate the bot version if ( !options.version || typeof options.version !== 'string' || !/^\d+(\.\d+){0,2}$/.test(options.version) ) { - throw new Error('Version must be a string formatted as #, #.#, or #.#.#'); + throw new Errors.ConfigError( + 'Version must be a string formatted as #, #.#, or #.#.#', + ); } + // Validate the policy URL try { if (options.policyURL) { new URL(options.policyURL); } } catch (err) { - throw new Error('Invalid policy URL'); + throw new Errors.ConfigError('Invalid policy URL'); } + // Ensure the minimum and maximum request delays are valid + if ( + options.minimumRequestDelay && + (typeof options.minimumRequestDelay !== 'number' || + options.minimumRequestDelay < 0) + ) { + throw new Errors.ConfigError( + 'Minimum request delay must be a positive number', + ); + } else if ( + options.maximumRequestDelay && + (typeof options.maximumRequestDelay !== 'number' || + options.maximumRequestDelay < 0) + ) { + throw new Errors.ConfigError( + 'Maximum request delay must be a positive number', + ); + } else if ( + options.minimumRequestDelay && + options.maximumRequestDelay && + options.minimumRequestDelay > options.maximumRequestDelay + ) { + throw new Errors.ConfigError( + 'Minimum request delay cannot be greater than maximum request delay', + ); + } + + // Set the options this.options = options; + // Set the bot name and user agent this.botName = options.name; this.userAgent = options.userAgent ?? `${this.botName}/${options.version} (+${ - options.policyURL ?? 'https://bit.ly/engine-source' + options.policyURL ?? 'https://npm.im/netscrape' })${options.hideLibraryAgent ? '' : ' NetScrape/0.1'}`; + // Initialize the request cache and DNS cache this.cache = new QuickLRU({ maxSize: 50 }); this.dnsCachable = new CacheableLookup({ cache: new QuickLRU({ maxSize: 1000 }), }); + + // Set the DNS servers this.dnsCachable.servers = [ '1.1.1.1', '[2606:4700:4700::1111]', @@ -92,10 +131,13 @@ class Bot { rawURL: string, options?: BotRequestOptions, ): Promise> { + // Parse URL const url = new URL(rawURL); + // Initialize the headers const standardHeaders = { ...options?.headers }; + // Convert all headers to lowercase for (const key of Object.keys(standardHeaders)) { if (key !== key.toLowerCase()) { standardHeaders[key.toLowerCase()] = standardHeaders[key]; @@ -103,6 +145,7 @@ class Bot { } } + // Initialize the default request options const defaultOptions: Partial = { timeout: { lookup: 6e4, @@ -118,7 +161,7 @@ class Bot { 'user-agent': this.userAgent, accept: 'text/html;q=0.9,image/webp,*/*;q=0.8', 'accept-language': 'en-US,en;q=0.5', - 'cache-control': 'max-age=0', + 'cache-control': 'max-age=86400', host: url.host, referer: url.origin, 'sec-fetch-dest': 'document', @@ -130,18 +173,21 @@ class Bot { 'upgrade-insecure-requests': '1', ...standardHeaders, }, - cache: this.cache, + cache: this.options.disableCaching ? false : this.cache, cacheOptions: { shared: false, immutableMinTimeToLive: 3600 * 1000, }, - dnsCache: this.dnsCachable, + dnsCache: this.options.disableCaching ? false : this.dnsCachable, + ...options?.overrides, }; + // Fetch the URL as a stream if requested if (options?.stream === true) { return got.stream(rawURL, { ...defaultOptions, isStream: true }); } + // Fetch the URL as a string if requested return got.get(rawURL, { ...defaultOptions, responseType: 'text', @@ -149,67 +195,93 @@ class Bot { } private async fetchRobotsTxt(origin: string) { - const robotsTxtURL = `${origin}/robots.txt`; + // Fetch the robots.txt + const robotsTxt = await this.fetchURL(`${origin}/robots.txt`, { + overrides: { throwHttpErrors: false }, + }); - try { - const robotsTxt = await this.fetchURL(robotsTxtURL); + if (robotsTxt.statusCode >= 400 && robotsTxt.statusCode < 500) { + // RFC 9309 2.3.1.3, allow all for 400 errors + this.robotsTxt[origin] = new RobotsTxt('User-agent: *\nDisallow:'); - this.robotsTxtExpires[origin] = new Date( - new Date().getTime() + 24 * 60 * 60 * 1000, - ); - this.robotsTxt[origin] = new RobotsTxt(robotsTxt.body); - } catch (err) { - console.error(err); + // 400 Errors are not cached + return; + } else if (robotsTxt.statusCode >= 500) { + // RFC 9309 2.3.1.4, must reject 500 errors + throw new Errors.RobotsRejection('Robots.txt server error'); + } + + // Parse the robots.txt + this.robotsTxt[origin] = new RobotsTxt(robotsTxt.body); + + // Wait for the required delay if not cached + if (!robotsTxt.isFromCache) { + await this.waitForRequestDelay(origin); } } private getDelay(origin: string) { + // Establish a minimum request delay (default 1 second) const minimumRequestDelay = this.options.minimumRequestDelay ?? 1000; - return (this.requestDelay[origin] ?? 0) > minimumRequestDelay - ? this.requestDelay[origin] - : minimumRequestDelay; + + // Calculate the delay for the origin based on the robots.txt + return Math.max(this.requestDelay[origin] ?? 0, minimumRequestDelay); + } + + private async waitForRequestDelay(origin: string) { + // Get the delay for the origin + const delay = this.getDelay(origin); + + // Calculate the wait time for the origin + const waitTime = + delay - (Date.now() - (this.requestTime[origin]?.getTime() ?? 0)); + + // Wait for the required delay + if (this.requestTime[origin] && waitTime > 0) { + // Check if the wait time is too long + if (waitTime <= (this.options.maximumRequestDelay ?? 10000)) { + await new Promise((res) => { + setTimeout(res, waitTime); + }); + } else { + throw new Errors.DelayError('Wait time too long'); + } + } + + // Set the request time for the next delay + this.requestTime[origin] = new Date(); } makeRequest(rawURL: string, asStream?: false): Promise>; makeRequest(rawURL: string, asStream: true): Promise; async makeRequest(rawURL: string, asStream = false) { + // Parse URL const url = new URL(rawURL); + // Get the robots.txt for the origin await this.fetchRobotsTxt(url.origin); + // Check if the path is allowed const allowedByRobots = this.robotsTxt[url.origin].isPathAllowed( `${url.pathname}${url.search}`, this.botName, ); + // If not allowed, throw a rejection if (!allowedByRobots) { - throw new APIError.BadGateway('Request blocked by robots.txt'); + throw new Errors.RobotsRejection('Request blocked by robots.txt'); } - const delay = this.getDelay(url.origin); - const waitTime = - delay - (Date.now() - (this.requestTime[url.origin]?.getTime() ?? 0)); - - if (this.requestTime[url.origin] && waitTime > 0) { - if (waitTime <= 10000) { - await new Promise((res) => { - setTimeout(res, waitTime); - }); - } else { - console.error('Wait time rejected'); - throw new APIError.BadGateway('Wait time too long'); - } - } - - this.requestTime[url.origin] = new Date(); + // Wait for the required delay + await this.waitForRequestDelay(url.origin); + // Fetch the URL as a stream if requested if (asStream) { return this.fetchURL(rawURL, { stream: true }); } - const content = await this.fetchURL(rawURL); - - return content; + // Fetch the URL as a string if requested + return this.fetchURL(rawURL); } } diff --git a/packages/bot/src/test/_server.util.ts b/packages/bot/src/test/_server.util.ts index 06fd67a..4a4e512 100644 --- a/packages/bot/src/test/_server.util.ts +++ b/packages/bot/src/test/_server.util.ts @@ -4,7 +4,10 @@ import { type Server } from 'http'; let currentPort = 3000; const getPort = () => currentPort++; -export function createServer(robotsTxt: string) { +export function createServer( + robotsTxt: string, + options?: { robots400?: boolean; robots500?: boolean }, +) { const app = express(); const requests: Request[] = []; app.get('/', (req, res) => { @@ -25,13 +28,22 @@ export function createServer(robotsTxt: string) { }); app.get('/robots.txt', (req, res) => { requests.push(req); + + if (options?.robots400) return res.status(404).send('Not Found'); + if (options?.robots500) + return res.status(500).send('Internal Server Error'); + + res.set('cache-control', 'public, max-age=86400'); res.send(robotsTxt); }); return { app, requests }; } -export async function startServer(robotsTxt: string) { - const { app, requests } = createServer(robotsTxt); +export async function startServer( + robotsTxt: string, + options?: Parameters[1], +) { + const { app, requests } = createServer(robotsTxt, options); const port = getPort(); const server = await new Promise((res) => { const s = app.listen(port, () => { diff --git a/packages/bot/src/test/bot.spec.ts b/packages/bot/src/test/bot.spec.ts index c89dce4..30c8ae7 100644 --- a/packages/bot/src/test/bot.spec.ts +++ b/packages/bot/src/test/bot.spec.ts @@ -1,5 +1,6 @@ import test from 'ava'; import Bot from '../index.js'; +import * as Errors from '../errors.js'; import { type Server } from 'http'; import { startServer, stopServer } from './_server.util.js'; @@ -16,29 +17,36 @@ test('Bot disallow none', async (it) => { const bot = new Bot({ name: 'Test', version: '0.1' }); + const ts1 = Date.now(); let req = await bot.makeRequest(`http://127.0.0.1:${start.port}/`); it.is(req.body, 'Home'); + const ts2 = Date.now(); req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a`); it.is(req.body, 'A'); + const ts3 = Date.now(); req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`); it.is(req.body, 'B'); + const ts4 = Date.now(); req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`); it.is(req.body, 'C'); - const [req1, req2, req3, req4, req5] = start.requests; - - console.log('Cache size', bot.cache.size); + const ts5 = Date.now(); - console.log(req1); + const [req1, req2, req3, req4, req5] = start.requests; it.is(req1.path, '/robots.txt'); it.is(req2.path, '/'); it.is(req3.path, '/a'); it.is(req4.path, '/a/b'); it.is(req5.path, '/a/b/c'); + + it.true(ts2 - ts1 >= 900 && ts2 - ts1 <= 1200); + it.true(ts3 - ts2 >= 900 && ts3 - ts2 <= 1200); + it.true(ts4 - ts3 >= 900 && ts4 - ts3 <= 1200); + it.true(ts5 - ts4 >= 900 && ts5 - ts4 <= 1200); }); test('Bot allow all', async (it) => { @@ -66,20 +74,98 @@ test('Bot disallow all', async (it) => { const bot = new Bot({ name: 'Test', version: '0.1' }); - await it.throwsAsync(() => - bot.makeRequest(`http://127.0.0.1:${start.port}/`), + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, + ); + + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/a`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, + ); + + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, + ); + + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, + ); +}); + +test('Bot responds to 400 errors', async (it) => { + const start = await startServer('User-agent: *\nAllow: /', { + robots400: true, + }); + server = start.server; + + const bot = new Bot({ name: 'Test', version: '0.1' }); + + let req = await bot.makeRequest(`http://127.0.0.1:${start.port}/`); + it.is(req.body, 'Home'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a`); + it.is(req.body, 'A'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`); + it.is(req.body, 'B'); + + req = await bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`); + it.is(req.body, 'C'); +}); + +test('Bot responds to 500 errors', async (it) => { + const start = await startServer('User-agent: *\nAllow: /', { + robots500: true, + }); + server = start.server; + + const bot = new Bot({ name: 'Test', version: '0.1' }); + + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, ); - await it.throwsAsync(() => - bot.makeRequest(`http://127.0.0.1:${start.port}/a`), + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/a`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, ); - await it.throwsAsync(() => - bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`), + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, ); - await it.throwsAsync(() => - bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`), + await it.throwsAsync( + () => bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`), + { + instanceOf: Errors.RobotsRejection, + message: 'Request blocked by robots.txt', + }, ); }); From 9fbd3e97e6f11f1880c257fa48abab3943b7a089 Mon Sep 17 00:00:00 2001 From: Russell Steadman Date: Thu, 12 Oct 2023 00:33:56 +0000 Subject: [PATCH 3/4] Add rfc compliance examples to exclusion --- packages/bot/README.md | 2 +- packages/exclusion/README.md | 4 +- packages/exclusion/package-lock.json | 222 ++++++++++-------- packages/exclusion/package.json | 8 +- packages/exclusion/src/index.ts | 32 ++- packages/exclusion/src/test/_test.util.ts | 5 +- .../src/test/assets/amazonRobots.json | 1 - .../src/test/assets/amazonRobots.txt | 155 ++++++++++++ .../exclusion/src/test/assets/rfc1Robots.txt | 19 ++ .../exclusion/src/test/assets/rfc2Robots.txt | 3 + .../exclusion/src/test/assets/rfc3Robots.txt | 3 + .../exclusion/src/test/assets/rfc4Robots.txt | 3 + packages/exclusion/src/test/main.spec.ts | 9 +- .../exclusion/src/test/rfcExamples.spec.ts | 108 +++++++++ 14 files changed, 448 insertions(+), 126 deletions(-) delete mode 100644 packages/exclusion/src/test/assets/amazonRobots.json create mode 100644 packages/exclusion/src/test/assets/amazonRobots.txt create mode 100644 packages/exclusion/src/test/assets/rfc1Robots.txt create mode 100644 packages/exclusion/src/test/assets/rfc2Robots.txt create mode 100644 packages/exclusion/src/test/assets/rfc3Robots.txt create mode 100644 packages/exclusion/src/test/assets/rfc4Robots.txt create mode 100644 packages/exclusion/src/test/rfcExamples.spec.ts diff --git a/packages/bot/README.md b/packages/bot/README.md index b94e757..d1021d2 100644 --- a/packages/bot/README.md +++ b/packages/bot/README.md @@ -4,5 +4,5 @@ Web scraping made efficient, simple, and compliant. ## License -MIT (C) 2022 [Russell Steadman](https://github.com/russellsteadman). See LICENSE file. Visit [Google +MIT (C) 2023 [Russell Steadman](https://github.com/russellsteadman). See LICENSE file. Visit [Google deps.dev](https://deps.dev/npm/netscrape) for dependency information. diff --git a/packages/exclusion/README.md b/packages/exclusion/README.md index 3b669ab..8295501 100644 --- a/packages/exclusion/README.md +++ b/packages/exclusion/README.md @@ -1,6 +1,6 @@ # Exclusion -A simple and compliant `robots.txt` parser for Node.js. +A simple and RFC 9309 compliant `robots.txt` parser for Node.js. ![](https://img.shields.io/librariesio/release/npm/exclusion?style=flat-square) ![](https://img.shields.io/npm/l/exclusion?style=flat-square) ![](https://img.shields.io/snyk/vulnerabilities/npm/exclusion?style=flat-square) @@ -42,5 +42,5 @@ robotsTxt.getDelay('MyUserAgent'); ## License -MIT (C) 2022 [Russell Steadman](https://github.com/russellsteadman). See LICENSE file. Visit [Google +MIT (C) 2023 [Russell Steadman](https://github.com/russellsteadman). See LICENSE file. Visit [Google deps.dev](https://deps.dev/npm/exclusion) for dependency license information. diff --git a/packages/exclusion/package-lock.json b/packages/exclusion/package-lock.json index c3e74fd..1f71234 100644 --- a/packages/exclusion/package-lock.json +++ b/packages/exclusion/package-lock.json @@ -1,33 +1,33 @@ { "name": "exclusion", - "version": "0.1.1", + "version": "0.2.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "exclusion", - "version": "0.1.1", + "version": "0.2.0", "license": "MIT", "dependencies": { - "@types/node": "^20.8.3" + "@types/node": "^20.8.4" }, "devDependencies": { "@ava/typescript": "^4.1.0", "@swc/cli": "^0.1.62", "@swc/core": "^1.3.92", - "@typescript-eslint/eslint-plugin": "^6.7.4", - "@typescript-eslint/parser": "^6.7.4", + "@typescript-eslint/eslint-plugin": "^6.7.5", + "@typescript-eslint/parser": "^6.7.5", "ava": "^5.3.1", "browserslist": "^4.22.1", "c8": "^8.0.1", "eslint": "^8.51.0", "eslint-plugin-ava": "^14.0.0", - "eslint-plugin-prettier": "^5.0.0", + "eslint-plugin-prettier": "^5.0.1", "prettier": "^3.0.3", "typescript": "^5.2.2" }, "engines": { - "node": ">=14.16" + "node": ">=18.18" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -694,9 +694,12 @@ } }, "node_modules/@types/node": { - "version": "20.8.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.3.tgz", - "integrity": "sha512-jxiZQFpb+NlH5kjW49vXxvxTjeeqlbsnTAdBTKpzEdPs9itay7MscYXz3Fo9VYFEsfQ6LJFitHad3faerLAjCw==" + "version": "20.8.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.4.tgz", + "integrity": "sha512-ZVPnqU58giiCjSxjVUESDtdPk4QR5WQhhINbc9UBrKLU68MX5BF6kbQzTrkwbolyr0X8ChBpXfavr5mZFKZQ5A==", + "dependencies": { + "undici-types": "~5.25.1" + } }, "node_modules/@types/responselike": { "version": "1.0.1", @@ -714,16 +717,16 @@ "dev": true }, "node_modules/@typescript-eslint/eslint-plugin": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.4.tgz", - "integrity": "sha512-DAbgDXwtX+pDkAHwiGhqP3zWUGpW49B7eqmgpPtg+BKJXwdct79ut9+ifqOFPJGClGKSHXn2PTBatCnldJRUoA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.5.tgz", + "integrity": "sha512-JhtAwTRhOUcP96D0Y6KYnwig/MRQbOoLGXTON2+LlyB/N35SP9j1boai2zzwXb7ypKELXMx3DVk9UTaEq1vHEw==", "dev": true, "dependencies": { "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.7.4", - "@typescript-eslint/type-utils": "6.7.4", - "@typescript-eslint/utils": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4", + "@typescript-eslint/scope-manager": "6.7.5", + "@typescript-eslint/type-utils": "6.7.5", + "@typescript-eslint/utils": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.4", @@ -749,15 +752,15 @@ } }, "node_modules/@typescript-eslint/parser": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.7.4.tgz", - "integrity": "sha512-I5zVZFY+cw4IMZUeNCU7Sh2PO5O57F7Lr0uyhgCJmhN/BuTlnc55KxPonR4+EM3GBdfiCyGZye6DgMjtubQkmA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.7.5.tgz", + "integrity": "sha512-bIZVSGx2UME/lmhLcjdVc7ePBwn7CLqKarUBL4me1C5feOd663liTGjMBGVcGr+BhnSLeP4SgwdvNnnkbIdkCw==", "dev": true, "dependencies": { - "@typescript-eslint/scope-manager": "6.7.4", - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/typescript-estree": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4", + "@typescript-eslint/scope-manager": "6.7.5", + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/typescript-estree": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5", "debug": "^4.3.4" }, "engines": { @@ -777,13 +780,13 @@ } }, "node_modules/@typescript-eslint/scope-manager": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.7.4.tgz", - "integrity": "sha512-SdGqSLUPTXAXi7c3Ob7peAGVnmMoGzZ361VswK2Mqf8UOYcODiYvs8rs5ILqEdfvX1lE7wEZbLyELCW+Yrql1A==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.7.5.tgz", + "integrity": "sha512-GAlk3eQIwWOJeb9F7MKQ6Jbah/vx1zETSDw8likab/eFcqkjSD7BI75SDAeC5N2L0MmConMoPvTsmkrg71+B1A==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4" + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5" }, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -794,13 +797,13 @@ } }, "node_modules/@typescript-eslint/type-utils": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.7.4.tgz", - "integrity": "sha512-n+g3zi1QzpcAdHFP9KQF+rEFxMb2KxtnJGID3teA/nxKHOVi3ylKovaqEzGBbVY2pBttU6z85gp0D00ufLzViQ==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.7.5.tgz", + "integrity": "sha512-Gs0qos5wqxnQrvpYv+pf3XfcRXW6jiAn9zE/K+DlmYf6FcpxeNYN0AIETaPR7rHO4K2UY+D0CIbDP9Ut0U4m1g==", "dev": true, "dependencies": { - "@typescript-eslint/typescript-estree": "6.7.4", - "@typescript-eslint/utils": "6.7.4", + "@typescript-eslint/typescript-estree": "6.7.5", + "@typescript-eslint/utils": "6.7.5", "debug": "^4.3.4", "ts-api-utils": "^1.0.1" }, @@ -821,9 +824,9 @@ } }, "node_modules/@typescript-eslint/types": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.4.tgz", - "integrity": "sha512-o9XWK2FLW6eSS/0r/tgjAGsYasLAnOWg7hvZ/dGYSSNjCh+49k5ocPN8OmG5aZcSJ8pclSOyVKP2x03Sj+RrCA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.5.tgz", + "integrity": "sha512-WboQBlOXtdj1tDFPyIthpKrUb+kZf2VroLZhxKa/VlwLlLyqv/PwUNgL30BlTVZV1Wu4Asu2mMYPqarSO4L5ZQ==", "dev": true, "engines": { "node": "^16.0.0 || >=18.0.0" @@ -834,13 +837,13 @@ } }, "node_modules/@typescript-eslint/typescript-estree": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.4.tgz", - "integrity": "sha512-ty8b5qHKatlNYd9vmpHooQz3Vki3gG+3PchmtsA4TgrZBKWHNjWfkQid7K7xQogBqqc7/BhGazxMD5vr6Ha+iQ==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.5.tgz", + "integrity": "sha512-NhJiJ4KdtwBIxrKl0BqG1Ur+uw7FiOnOThcYx9DpOGJ/Abc9z2xNzLeirCG02Ig3vkvrc2qFLmYSSsaITbKjlg==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4", + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -861,17 +864,17 @@ } }, "node_modules/@typescript-eslint/utils": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.7.4.tgz", - "integrity": "sha512-PRQAs+HUn85Qdk+khAxsVV+oULy3VkbH3hQ8hxLRJXWBEd7iI+GbQxH5SEUSH7kbEoTp6oT1bOwyga24ELALTA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.7.5.tgz", + "integrity": "sha512-pfRRrH20thJbzPPlPc4j0UNGvH1PjPlhlCMq4Yx7EGjV7lvEeGX0U6MJYe8+SyFutWgSHsdbJ3BXzZccYggezA==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.7.4", - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/typescript-estree": "6.7.4", + "@typescript-eslint/scope-manager": "6.7.5", + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/typescript-estree": "6.7.5", "semver": "^7.5.4" }, "engines": { @@ -886,12 +889,12 @@ } }, "node_modules/@typescript-eslint/visitor-keys": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.4.tgz", - "integrity": "sha512-pOW37DUhlTZbvph50x5zZCkFn3xzwkGtNoJHzIM3svpiSkJzwOYr/kVBaXmf+RAQiUDs1AHEZVNPg6UJCJpwRA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.5.tgz", + "integrity": "sha512-3MaWdDZtLlsexZzDSdQWsFQ9l9nL8B80Z4fImSpyllFC/KLqWQRdEcB+gGGO+N3Q2uL40EsG66wZLsohPxNXvg==", "dev": true, "dependencies": { - "@typescript-eslint/types": "6.7.4", + "@typescript-eslint/types": "6.7.5", "eslint-visitor-keys": "^3.4.1" }, "engines": { @@ -2212,9 +2215,9 @@ } }, "node_modules/eslint-plugin-prettier": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.0.tgz", - "integrity": "sha512-AgaZCVuYDXHUGxj/ZGu1u8H8CYgDY3iG6w5kUFw4AzMVXzB7VvbKgYR4nATIN+OvUrghMbiDLeimVjVY5ilq3w==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.1.tgz", + "integrity": "sha512-m3u5RnR56asrwV/lDC4GHorlW75DsFfmUcjfCYylTUs85dBRnB7VM6xG8eCMJdeDRnppzmxZVf1GEPJvl1JmNg==", "dev": true, "dependencies": { "prettier-linter-helpers": "^1.0.0", @@ -4850,6 +4853,11 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "5.25.3", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.25.3.tgz", + "integrity": "sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==" + }, "node_modules/untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -5586,9 +5594,12 @@ } }, "@types/node": { - "version": "20.8.3", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.3.tgz", - "integrity": "sha512-jxiZQFpb+NlH5kjW49vXxvxTjeeqlbsnTAdBTKpzEdPs9itay7MscYXz3Fo9VYFEsfQ6LJFitHad3faerLAjCw==" + "version": "20.8.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.8.4.tgz", + "integrity": "sha512-ZVPnqU58giiCjSxjVUESDtdPk4QR5WQhhINbc9UBrKLU68MX5BF6kbQzTrkwbolyr0X8ChBpXfavr5mZFKZQ5A==", + "requires": { + "undici-types": "~5.25.1" + } }, "@types/responselike": { "version": "1.0.1", @@ -5606,16 +5617,16 @@ "dev": true }, "@typescript-eslint/eslint-plugin": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.4.tgz", - "integrity": "sha512-DAbgDXwtX+pDkAHwiGhqP3zWUGpW49B7eqmgpPtg+BKJXwdct79ut9+ifqOFPJGClGKSHXn2PTBatCnldJRUoA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.7.5.tgz", + "integrity": "sha512-JhtAwTRhOUcP96D0Y6KYnwig/MRQbOoLGXTON2+LlyB/N35SP9j1boai2zzwXb7ypKELXMx3DVk9UTaEq1vHEw==", "dev": true, "requires": { "@eslint-community/regexpp": "^4.5.1", - "@typescript-eslint/scope-manager": "6.7.4", - "@typescript-eslint/type-utils": "6.7.4", - "@typescript-eslint/utils": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4", + "@typescript-eslint/scope-manager": "6.7.5", + "@typescript-eslint/type-utils": "6.7.5", + "@typescript-eslint/utils": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5", "debug": "^4.3.4", "graphemer": "^1.4.0", "ignore": "^5.2.4", @@ -5625,54 +5636,54 @@ } }, "@typescript-eslint/parser": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.7.4.tgz", - "integrity": "sha512-I5zVZFY+cw4IMZUeNCU7Sh2PO5O57F7Lr0uyhgCJmhN/BuTlnc55KxPonR4+EM3GBdfiCyGZye6DgMjtubQkmA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.7.5.tgz", + "integrity": "sha512-bIZVSGx2UME/lmhLcjdVc7ePBwn7CLqKarUBL4me1C5feOd663liTGjMBGVcGr+BhnSLeP4SgwdvNnnkbIdkCw==", "dev": true, "requires": { - "@typescript-eslint/scope-manager": "6.7.4", - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/typescript-estree": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4", + "@typescript-eslint/scope-manager": "6.7.5", + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/typescript-estree": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5", "debug": "^4.3.4" } }, "@typescript-eslint/scope-manager": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.7.4.tgz", - "integrity": "sha512-SdGqSLUPTXAXi7c3Ob7peAGVnmMoGzZ361VswK2Mqf8UOYcODiYvs8rs5ILqEdfvX1lE7wEZbLyELCW+Yrql1A==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-6.7.5.tgz", + "integrity": "sha512-GAlk3eQIwWOJeb9F7MKQ6Jbah/vx1zETSDw8likab/eFcqkjSD7BI75SDAeC5N2L0MmConMoPvTsmkrg71+B1A==", "dev": true, "requires": { - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4" + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5" } }, "@typescript-eslint/type-utils": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.7.4.tgz", - "integrity": "sha512-n+g3zi1QzpcAdHFP9KQF+rEFxMb2KxtnJGID3teA/nxKHOVi3ylKovaqEzGBbVY2pBttU6z85gp0D00ufLzViQ==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-6.7.5.tgz", + "integrity": "sha512-Gs0qos5wqxnQrvpYv+pf3XfcRXW6jiAn9zE/K+DlmYf6FcpxeNYN0AIETaPR7rHO4K2UY+D0CIbDP9Ut0U4m1g==", "dev": true, "requires": { - "@typescript-eslint/typescript-estree": "6.7.4", - "@typescript-eslint/utils": "6.7.4", + "@typescript-eslint/typescript-estree": "6.7.5", + "@typescript-eslint/utils": "6.7.5", "debug": "^4.3.4", "ts-api-utils": "^1.0.1" } }, "@typescript-eslint/types": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.4.tgz", - "integrity": "sha512-o9XWK2FLW6eSS/0r/tgjAGsYasLAnOWg7hvZ/dGYSSNjCh+49k5ocPN8OmG5aZcSJ8pclSOyVKP2x03Sj+RrCA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-6.7.5.tgz", + "integrity": "sha512-WboQBlOXtdj1tDFPyIthpKrUb+kZf2VroLZhxKa/VlwLlLyqv/PwUNgL30BlTVZV1Wu4Asu2mMYPqarSO4L5ZQ==", "dev": true }, "@typescript-eslint/typescript-estree": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.4.tgz", - "integrity": "sha512-ty8b5qHKatlNYd9vmpHooQz3Vki3gG+3PchmtsA4TgrZBKWHNjWfkQid7K7xQogBqqc7/BhGazxMD5vr6Ha+iQ==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-6.7.5.tgz", + "integrity": "sha512-NhJiJ4KdtwBIxrKl0BqG1Ur+uw7FiOnOThcYx9DpOGJ/Abc9z2xNzLeirCG02Ig3vkvrc2qFLmYSSsaITbKjlg==", "dev": true, "requires": { - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/visitor-keys": "6.7.4", + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/visitor-keys": "6.7.5", "debug": "^4.3.4", "globby": "^11.1.0", "is-glob": "^4.0.3", @@ -5681,27 +5692,27 @@ } }, "@typescript-eslint/utils": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.7.4.tgz", - "integrity": "sha512-PRQAs+HUn85Qdk+khAxsVV+oULy3VkbH3hQ8hxLRJXWBEd7iI+GbQxH5SEUSH7kbEoTp6oT1bOwyga24ELALTA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-6.7.5.tgz", + "integrity": "sha512-pfRRrH20thJbzPPlPc4j0UNGvH1PjPlhlCMq4Yx7EGjV7lvEeGX0U6MJYe8+SyFutWgSHsdbJ3BXzZccYggezA==", "dev": true, "requires": { "@eslint-community/eslint-utils": "^4.4.0", "@types/json-schema": "^7.0.12", "@types/semver": "^7.5.0", - "@typescript-eslint/scope-manager": "6.7.4", - "@typescript-eslint/types": "6.7.4", - "@typescript-eslint/typescript-estree": "6.7.4", + "@typescript-eslint/scope-manager": "6.7.5", + "@typescript-eslint/types": "6.7.5", + "@typescript-eslint/typescript-estree": "6.7.5", "semver": "^7.5.4" } }, "@typescript-eslint/visitor-keys": { - "version": "6.7.4", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.4.tgz", - "integrity": "sha512-pOW37DUhlTZbvph50x5zZCkFn3xzwkGtNoJHzIM3svpiSkJzwOYr/kVBaXmf+RAQiUDs1AHEZVNPg6UJCJpwRA==", + "version": "6.7.5", + "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-6.7.5.tgz", + "integrity": "sha512-3MaWdDZtLlsexZzDSdQWsFQ9l9nL8B80Z4fImSpyllFC/KLqWQRdEcB+gGGO+N3Q2uL40EsG66wZLsohPxNXvg==", "dev": true, "requires": { - "@typescript-eslint/types": "6.7.4", + "@typescript-eslint/types": "6.7.5", "eslint-visitor-keys": "^3.4.1" } }, @@ -6654,9 +6665,9 @@ } }, "eslint-plugin-prettier": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.0.tgz", - "integrity": "sha512-AgaZCVuYDXHUGxj/ZGu1u8H8CYgDY3iG6w5kUFw4AzMVXzB7VvbKgYR4nATIN+OvUrghMbiDLeimVjVY5ilq3w==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-5.0.1.tgz", + "integrity": "sha512-m3u5RnR56asrwV/lDC4GHorlW75DsFfmUcjfCYylTUs85dBRnB7VM6xG8eCMJdeDRnppzmxZVf1GEPJvl1JmNg==", "dev": true, "requires": { "prettier-linter-helpers": "^1.0.0", @@ -8403,6 +8414,11 @@ "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", "dev": true }, + "undici-types": { + "version": "5.25.3", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.25.3.tgz", + "integrity": "sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==" + }, "untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", diff --git a/packages/exclusion/package.json b/packages/exclusion/package.json index 40e5c45..34de066 100644 --- a/packages/exclusion/package.json +++ b/packages/exclusion/package.json @@ -23,7 +23,7 @@ "prepublishOnly": "npm run build && ava" }, "dependencies": { - "@types/node": "^20.8.3" + "@types/node": "^20.8.4" }, "repository": { "type": "git", @@ -49,14 +49,14 @@ "@ava/typescript": "^4.1.0", "@swc/cli": "^0.1.62", "@swc/core": "^1.3.92", - "@typescript-eslint/eslint-plugin": "^6.7.4", - "@typescript-eslint/parser": "^6.7.4", + "@typescript-eslint/eslint-plugin": "^6.7.5", + "@typescript-eslint/parser": "^6.7.5", "ava": "^5.3.1", "browserslist": "^4.22.1", "c8": "^8.0.1", "eslint": "^8.51.0", "eslint-plugin-ava": "^14.0.0", - "eslint-plugin-prettier": "^5.0.0", + "eslint-plugin-prettier": "^5.0.1", "prettier": "^3.0.3", "typescript": "^5.2.2" }, diff --git a/packages/exclusion/src/index.ts b/packages/exclusion/src/index.ts index 6dc0fa4..40037e9 100644 --- a/packages/exclusion/src/index.ts +++ b/packages/exclusion/src/index.ts @@ -60,6 +60,7 @@ export class RobotsTxtLine { } // Check if a user-agent rule applies + // RFC 9309 2.2.1: Case insensitive, * matches all isOwnUserAgent(self: string) { if (this.key !== 'user-agent') return AgentSpecificity.NotSpecified; if ( @@ -132,9 +133,25 @@ export class RobotsTxt { // order they occur in the record. The first match found is used. If no match // is found, the default assumption is that the URL is allowed. isPathAllowed(path: string, userAgent: string) { + if (path === '/robots.txt') return true; + + let robotsSpecificity: AgentSpecificity = AgentSpecificity.NotSpecified; + + // Step 1: Find the most specific user-agent rule or rules + for (const line of this.lines) { + robotsSpecificity = Math.max( + robotsSpecificity, + line.isOwnUserAgent(userAgent), + ); + } + + if (robotsSpecificity === AgentSpecificity.NotSpecified) return true; + + // Step 2: Find the most specific rule for the user-agent + let ownUserAgent: AgentSpecificity = AgentSpecificity.NotSpecified; - const status = new Map>(); + const status = new Map(); for (const line of this.lines) { if (line.key === 'user-agent') { @@ -147,24 +164,19 @@ export class RobotsTxt { (line.key === 'allow' || line.key === 'disallow') && ownUserAgent ) { + if (ownUserAgent < robotsSpecificity) continue; + const lineStatus = line.isPathAllowedByLine(path); if ( typeof lineStatus === 'boolean' && typeof line.priority === 'number' ) { - const lineResult = - status.get(ownUserAgent) ?? new Map(); - if (!lineResult.has(line.priority)) - lineResult.set(line.priority, lineStatus); - status.set(ownUserAgent, lineResult); + status.set(line.priority, lineStatus || !!status.get(line.priority)); } } } - const maxStatus = - status.get(Math.max(...status.keys())) ?? new Map(); - - return maxStatus.get(Math.max(...maxStatus.keys())) ?? true; + return status.get(Math.max(...status.keys())) ?? true; } getDelay(userAgent: string) { diff --git a/packages/exclusion/src/test/_test.util.ts b/packages/exclusion/src/test/_test.util.ts index 8a16281..7224785 100644 --- a/packages/exclusion/src/test/_test.util.ts +++ b/packages/exclusion/src/test/_test.util.ts @@ -1,5 +1,8 @@ import fs from 'fs'; import { resolve } from 'path'; +export const loadText = (path: string) => + fs.readFileSync(resolve('src/test', path)).toString(); + export const loadJSON = (path: string) => - JSON.parse(fs.readFileSync(resolve('src/test', path)).toString()) as T; + JSON.parse(loadText(path)) as T; diff --git a/packages/exclusion/src/test/assets/amazonRobots.json b/packages/exclusion/src/test/assets/amazonRobots.json deleted file mode 100644 index 7850714..0000000 --- a/packages/exclusion/src/test/assets/amazonRobots.json +++ /dev/null @@ -1 +0,0 @@ -"User-agent: *\nCrawl-Delay: 5\nDisallow: /exec/obidos/account-access-login\nDisallow: /exec/obidos/change-style\nDisallow: /exec/obidos/flex-sign-in\nDisallow: /exec/obidos/handle-buy-box\nDisallow: /exec/obidos/tg/cm/member/\nDisallow: /gp/aw/help/id=sss\nDisallow: /gp/cart\nDisallow: /gp/flex\nDisallow: /gp/product/e-mail-friend\nDisallow: /gp/product/product-availability\nDisallow: /gp/product/rate-this-item\nDisallow: /gp/sign-in\nDisallow: /gp/reader\nDisallow: /gp/sitbv3/reader\nDisallow: /gp/richpub/syltguides/create\nDisallow: /gp/gfix\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/dmusic/order\nDisallow: /gp/legacy-handle-buy-box.html\nDisallow: /gp/aws/ssop\nDisallow: /gp/yourstore\nDisallow: /gp/gift-central/organizer/add-wishlist\nDisallow: /gp/vote\nDisallow: /gp/voting/\nDisallow: /gp/music/wma-pop-up\nDisallow: /gp/customer-images\nDisallow: /gp/richpub/listmania/createpipeline\nDisallow: /gp/content-form\nDisallow: /gp/pdp/invitation/invite\nDisallow: /gp/customer-reviews/common/du\nDisallow: /gp/customer-reviews/write-a-review.html\nDisallow: /gp/associations/wizard.html\nDisallow: /gp/music/clipserve\nDisallow: /gp/customer-media/upload\nDisallow: /gp/history\nDisallow: /gp/item-dispatch\nDisallow: /gp/dmusic/order/handle-buy-box.html\nDisallow: /gp/recsradio\nDisallow: /gp/slredirect\nDisallow: /dp/shipping/\nDisallow: /dp/twister-update/\nDisallow: /dp/manual-submit/\nDisallow: /dp/e-mail-friend/\nDisallow: /dp/product-availability/\nDisallow: /dp/rate-this-item/\nDisallow: /gp/registry/wishlist/*/reserve\nDisallow: /gp/structured-ratings/actions/get-experience.html\nDisallow: /gp/twitter/\nDisallow: /ap/signin\nDisallow: /gp/registry/wishlist/\nDisallow: /wishlist/\nAllow: /wishlist/universal*\nAllow: /wishlist/vendor-button*\nAllow: /wishlist/get-button*\nDisallow: /gp/wishlist/\nAllow: /gp/wishlist/universal*\nAllow: /gp/wishlist/vendor-button*\nAllow: /gp/wishlist/ipad-install*\nDisallow: /registry/wishlist/\nDisallow: /review/common/du\nDisallow: /gp/registry/search.html\nDisallow: /product-reviews/B0069IY63Y\nDisallow: /gp/orc/rml/\nDisallow: */gcrnsts\nDisallow: /gp/gc/widget\nDisallow: /gp/dmusic/mp3/player\nDisallow: /gp/entity-alert/external\nDisallow: /gp/customer-reviews/dynamic/sims-box\nDisallow: /review/dynamic/sims-box\nDisallow: /gp/redirect.html\nDisallow: /gp/twister/ajaxv2\nDisallow: /ss/twister/ajax\nDisallow: /b?*node=7454917011\nDisallow: /b?*node=7454927011\nDisallow: /b?*node=7454939011\nDisallow: /b?*node=7454898011\nDisallow: /gp/customer-media/actions/delete/\nDisallow: /gp/customer-media/actions/edit-caption/\nDisallow: /gp/dmusic/\nAllow: /gp/dmusic/promotions/PrimeMusic\nAllow: /gp/dmusic/promotions/AmazonMusicUnlimited\nDisallow: /gp/offer-listing/\nDisallow: /b?*node=9052533011\nDisallow: /lm/R1XIHQVKXSKBNJ\nDisallow: /lm/R3HQ5WJSZK6QSO\nDisallow: /surprise/\nDisallow: /local/ajax/\nDisallow: */B00M3E1NYI\nDisallow: */B00M3E1Q5Y\nDisallow: */B00M3E1TOM\nDisallow: */B00M3E1WYO\nDisallow: */B00M3E204K\nDisallow: */B00M3E236A\nDisallow: */B00M3E260I\nDisallow: */B00M3E28WO\nDisallow: */B00M3E2BC6\nDisallow: */B00M3E2DPQ\nDisallow: */B00M3E2GU8\nDisallow: */B00M3E2J14\nDisallow: */B00M3E2LOE\nDisallow: */B00M3E1HJY\nDisallow: /gp/socialmedia/giveaways\nDisallow: /gp/b2b-rd\nDisallow: /gp/aw/so.html\nDisallow: /gp/rentallist\nDisallow: /gp/video/dvd-rental/settings\nDisallow: /gp/rl/settings\nDisallow: /gp/video/settings\nDisallow: /gp/video/library\nDisallow: /gp/video/watchlist\nDisallow: /reviews/iframe\nDisallow: /gp/switch-language\nDisallow: /ga/p/\nDisallow: /gp/profile/\nDisallow: /giveaway/host/setup/\nDisallow: /ss/customer-reviews/lighthouse/\nDisallow: /ospublishing/story/*\nDisallow: /gp/aw/ol/\nDisallow: /gp/promotion/\nDisallow: /hz/leaderboard/top-reviewers/\nDisallow: /creatorhub\nDisallow: /creatorhub/*\nDisallow: /slp/s$\nDisallow: /-/\nAllow: /-/es/\nAllow: /-/zh_TW/\nAllow: /-/he/\nDisallow: /hz/help/contact/*/message/$\nDisallow: /gp/aw/shoppingAids/\nDisallow: /rss/people/*/reviews\nDisallow: /gp/pdp/rss/*/reviews\nDisallow: /gp/cdp/member-reviews/\nDisallow: /gp/aw/cr/\nDisallow: */sim/B001132UEE\nAllow: /gp/offer-listing/B000\nAllow: /gp/offer-listing/9000\nDisallow: /gp/aag\nAllow: /gp/aag/main?*seller=ABVFEJU8LS620\nDisallow: /gp/pdp/profile/\nDisallow: /gp/help/customer/express/c2c/\nDisallow: /slp/*/b$\nDisallow: /hz/contact-us/ajax/initiate-trusted-contact/\nDisallow: /gp/video/api\nDisallow: /hp/video/api\nDisallow: /gp/video/mystuff\nDisallow: /hp/video/mystuff\nDisallow: /gp/video/profiles\nDisallow: /hp/video/profiles\n\nUser-agent: EtaoSpider\nUser-Agent: Test_1\nCrawl-Delay:10\nDisallow: /" diff --git a/packages/exclusion/src/test/assets/amazonRobots.txt b/packages/exclusion/src/test/assets/amazonRobots.txt new file mode 100644 index 0000000..86f19f8 --- /dev/null +++ b/packages/exclusion/src/test/assets/amazonRobots.txt @@ -0,0 +1,155 @@ +User-agent: * +Crawl-Delay: 5 +Disallow: /exec/obidos/account-access-login +Disallow: /exec/obidos/change-style +Disallow: /exec/obidos/flex-sign-in +Disallow: /exec/obidos/handle-buy-box +Disallow: /exec/obidos/tg/cm/member/ +Disallow: /gp/aw/help/id=sss +Disallow: /gp/cart +Disallow: /gp/flex +Disallow: /gp/product/e-mail-friend +Disallow: /gp/product/product-availability +Disallow: /gp/product/rate-this-item +Disallow: /gp/sign-in +Disallow: /gp/reader +Disallow: /gp/sitbv3/reader +Disallow: /gp/richpub/syltguides/create +Disallow: /gp/gfix +Disallow: /gp/associations/wizard.html +Disallow: /gp/dmusic/order +Disallow: /gp/legacy-handle-buy-box.html +Disallow: /gp/aws/ssop +Disallow: /gp/yourstore +Disallow: /gp/gift-central/organizer/add-wishlist +Disallow: /gp/vote +Disallow: /gp/voting/ +Disallow: /gp/music/wma-pop-up +Disallow: /gp/customer-images +Disallow: /gp/richpub/listmania/createpipeline +Disallow: /gp/content-form +Disallow: /gp/pdp/invitation/invite +Disallow: /gp/customer-reviews/common/du +Disallow: /gp/customer-reviews/write-a-review.html +Disallow: /gp/associations/wizard.html +Disallow: /gp/music/clipserve +Disallow: /gp/customer-media/upload +Disallow: /gp/history +Disallow: /gp/item-dispatch +Disallow: /gp/dmusic/order/handle-buy-box.html +Disallow: /gp/recsradio +Disallow: /gp/slredirect +Disallow: /dp/shipping/ +Disallow: /dp/twister-update/ +Disallow: /dp/manual-submit/ +Disallow: /dp/e-mail-friend/ +Disallow: /dp/product-availability/ +Disallow: /dp/rate-this-item/ +Disallow: /gp/registry/wishlist/*/reserve +Disallow: /gp/structured-ratings/actions/get-experience.html +Disallow: /gp/twitter/ +Disallow: /ap/signin +Disallow: /gp/registry/wishlist/ +Disallow: /wishlist/ +Allow: /wishlist/universal* +Allow: /wishlist/vendor-button* +Allow: /wishlist/get-button* +Disallow: /gp/wishlist/ +Allow: /gp/wishlist/universal* +Allow: /gp/wishlist/vendor-button* +Allow: /gp/wishlist/ipad-install* +Disallow: /registry/wishlist/ +Disallow: /review/common/du +Disallow: /gp/registry/search.html +Disallow: /product-reviews/B0069IY63Y +Disallow: /gp/orc/rml/ +Disallow: */gcrnsts +Disallow: /gp/gc/widget +Disallow: /gp/dmusic/mp3/player +Disallow: /gp/entity-alert/external +Disallow: /gp/customer-reviews/dynamic/sims-box +Disallow: /review/dynamic/sims-box +Disallow: /gp/redirect.html +Disallow: /gp/twister/ajaxv2 +Disallow: /ss/twister/ajax +Disallow: /b?*node=7454917011 +Disallow: /b?*node=7454927011 +Disallow: /b?*node=7454939011 +Disallow: /b?*node=7454898011 +Disallow: /gp/customer-media/actions/delete/ +Disallow: /gp/customer-media/actions/edit-caption/ +Disallow: /gp/dmusic/ +Allow: /gp/dmusic/promotions/PrimeMusic +Allow: /gp/dmusic/promotions/AmazonMusicUnlimited +Disallow: /gp/offer-listing/ +Disallow: /b?*node=9052533011 +Disallow: /lm/R1XIHQVKXSKBNJ +Disallow: /lm/R3HQ5WJSZK6QSO +Disallow: /surprise/ +Disallow: /local/ajax/ +Disallow: */B00M3E1NYI +Disallow: */B00M3E1Q5Y +Disallow: */B00M3E1TOM +Disallow: */B00M3E1WYO +Disallow: */B00M3E204K +Disallow: */B00M3E236A +Disallow: */B00M3E260I +Disallow: */B00M3E28WO +Disallow: */B00M3E2BC6 +Disallow: */B00M3E2DPQ +Disallow: */B00M3E2GU8 +Disallow: */B00M3E2J14 +Disallow: */B00M3E2LOE +Disallow: */B00M3E1HJY +Disallow: /gp/socialmedia/giveaways +Disallow: /gp/b2b-rd +Disallow: /gp/aw/so.html +Disallow: /gp/rentallist +Disallow: /gp/video/dvd-rental/settings +Disallow: /gp/rl/settings +Disallow: /gp/video/settings +Disallow: /gp/video/library +Disallow: /gp/video/watchlist +Disallow: /reviews/iframe +Disallow: /gp/switch-language +Disallow: /ga/p/ +Disallow: /gp/profile/ +Disallow: /giveaway/host/setup/ +Disallow: /ss/customer-reviews/lighthouse/ +Disallow: /ospublishing/story/* +Disallow: /gp/aw/ol/ +Disallow: /gp/promotion/ +Disallow: /hz/leaderboard/top-reviewers/ +Disallow: /creatorhub +Disallow: /creatorhub/* +Disallow: /slp/s$ +Disallow: /-/ +Allow: /-/es/ +Allow: /-/zh_TW/ +Allow: /-/he/ +Disallow: /hz/help/contact/*/message/$ +Disallow: /gp/aw/shoppingAids/ +Disallow: /rss/people/*/reviews +Disallow: /gp/pdp/rss/*/reviews +Disallow: /gp/cdp/member-reviews/ +Disallow: /gp/aw/cr/ +Disallow: */sim/B001132UEE +Allow: /gp/offer-listing/B000 +Allow: /gp/offer-listing/9000 +Disallow: /gp/aag +Allow: /gp/aag/main?*seller=ABVFEJU8LS620 +Disallow: /gp/pdp/profile/ +Disallow: /gp/help/customer/express/c2c/ +Disallow: /slp/*/b$ +Disallow: /hz/contact-us/ajax/initiate-trusted-contact/ +Disallow: /gp/video/api +Disallow: /hp/video/api +Disallow: /gp/video/mystuff +Disallow: /hp/video/mystuff +Disallow: /gp/video/profiles +Disallow: /hp/video/profiles + +User-agent: EtaoSpider +User-Agent: Test_1 +Crawl-Delay:10 +Disallow: / \ No newline at end of file diff --git a/packages/exclusion/src/test/assets/rfc1Robots.txt b/packages/exclusion/src/test/assets/rfc1Robots.txt new file mode 100644 index 0000000..e872ef0 --- /dev/null +++ b/packages/exclusion/src/test/assets/rfc1Robots.txt @@ -0,0 +1,19 @@ +# robots.txt for http://www.example.com/ +User-Agent: * +Disallow: *.gif$ +Disallow: /example/ +Allow: /publications/ + +# Foobot is only allowed on specific pages +User-Agent: foobot +Disallow:/ +Allow:/example/page.html +Allow:/example/allowed.gif + +# Both barbot and bazbot apply +User-Agent: barbot +User-Agent: bazbot +Disallow: /example/page.html + +# Unrestricted +User-Agent: quxbot \ No newline at end of file diff --git a/packages/exclusion/src/test/assets/rfc2Robots.txt b/packages/exclusion/src/test/assets/rfc2Robots.txt new file mode 100644 index 0000000..41b8604 --- /dev/null +++ b/packages/exclusion/src/test/assets/rfc2Robots.txt @@ -0,0 +1,3 @@ +User-Agent: foobot +Allow: /example/page/ +Disallow: /example/page/disallowed.gif \ No newline at end of file diff --git a/packages/exclusion/src/test/assets/rfc3Robots.txt b/packages/exclusion/src/test/assets/rfc3Robots.txt new file mode 100644 index 0000000..bed6722 --- /dev/null +++ b/packages/exclusion/src/test/assets/rfc3Robots.txt @@ -0,0 +1,3 @@ +User-Agent: foobot +Disallow: /example/page/ +Allow: /example/page/ diff --git a/packages/exclusion/src/test/assets/rfc4Robots.txt b/packages/exclusion/src/test/assets/rfc4Robots.txt new file mode 100644 index 0000000..98154f2 --- /dev/null +++ b/packages/exclusion/src/test/assets/rfc4Robots.txt @@ -0,0 +1,3 @@ +User-Agent: foobot +Allow: /example/page/ +Disallow: /example/page/ diff --git a/packages/exclusion/src/test/main.spec.ts b/packages/exclusion/src/test/main.spec.ts index b379d02..d7a36cf 100644 --- a/packages/exclusion/src/test/main.spec.ts +++ b/packages/exclusion/src/test/main.spec.ts @@ -1,11 +1,12 @@ import test from 'ava'; import { RobotsTxt } from '../index.js'; -import { loadJSON } from './_test.util.js'; +import { loadJSON, loadText } from './_test.util.js'; +import fs from 'fs'; const { LINE_EQUAL_TEST_CASES } = loadJSON>( './assets/testCases.json', ); -const amazonRobots = loadJSON('./assets/amazonRobots.json'); +const amazonRobots = loadText('./assets/amazonRobots.txt'); const robotsTxtSourceOne = `User-Agent: Hello ${LINE_EQUAL_TEST_CASES.map(([line]) => `Disallow: ${line}`).join('\n')} @@ -53,11 +54,11 @@ test('RobotsTxt#isPathAllowed()', (it) => { it.is(robotsTxt.isPathAllowed(result, 'Goodbye'), false, result); } - it.is(robotsTxt.isPathAllowed('/random', 'Hello'), false); + it.is(robotsTxt.isPathAllowed('/random', 'Hello'), true); it.is(robotsTxt.isPathAllowed('/random', 'Not_an_agent'), false); it.is(robotsTxt.isPathAllowed('/random', 'Goodbye'), false); - it.is(robotsTxt.isPathAllowed('/', 'Hello'), false); + it.is(robotsTxt.isPathAllowed('/', 'Hello'), true); it.is(robotsTxt.isPathAllowed('/', 'Not_an_agent'), false); it.is(robotsTxt.isPathAllowed('/', 'Goodbye'), false); diff --git a/packages/exclusion/src/test/rfcExamples.spec.ts b/packages/exclusion/src/test/rfcExamples.spec.ts new file mode 100644 index 0000000..70bc617 --- /dev/null +++ b/packages/exclusion/src/test/rfcExamples.spec.ts @@ -0,0 +1,108 @@ +import test from 'ava'; +import { RobotsTxt } from '../index.js'; +import { loadText } from './_test.util.js'; + +const rfcOneRobots = loadText('./assets/rfc1Robots.txt'); +const rfcTwoRobots = loadText('./assets/rfc2Robots.txt'); +const rfcThreeRobots = loadText('./assets/rfc3Robots.txt'); +const rfcFourRobots = loadText('./assets/rfc4Robots.txt'); + +test('RFC 9309 5.1: Catch all', (it) => { + const robotsTxt = new RobotsTxt(rfcOneRobots); + + const ua = 'Test_1'; + + it.is(robotsTxt.isPathAllowed('/', ua), true); + it.is(robotsTxt.isPathAllowed('/foo', ua), true); + it.is(robotsTxt.isPathAllowed('/foo/bar', ua), true); + it.is(robotsTxt.isPathAllowed('/foo/bin.gif', ua), false); + it.is(robotsTxt.isPathAllowed('/foo/bin.gif/baz', ua), true); + it.is(robotsTxt.isPathAllowed('/example', ua), true); + it.is(robotsTxt.isPathAllowed('/example/', ua), false); + it.is(robotsTxt.isPathAllowed('/example/foo', ua), false); + it.is(robotsTxt.isPathAllowed('/publications/', ua), true); +}); + +test('RFC 9309 5.1: Simple Case', (it) => { + const robotsTxt = new RobotsTxt(rfcOneRobots); + + const ua = 'foobot'; + + it.is(robotsTxt.isPathAllowed('/', ua), false); + it.is(robotsTxt.isPathAllowed('/foo', ua), false); + it.is(robotsTxt.isPathAllowed('/foo/bar', ua), false); + it.is(robotsTxt.isPathAllowed('/example/page.html', ua), true); + it.is(robotsTxt.isPathAllowed('/example/page.html/test', ua), true); + it.is(robotsTxt.isPathAllowed('/example/allowed.gif', ua), true); + it.is(robotsTxt.isPathAllowed('/example/allowed.gif/xyz', ua), true); + it.is(robotsTxt.isPathAllowed('/example/', ua), false); + it.is(robotsTxt.isPathAllowed('/publications/', ua), false); +}); + +test('RFC 9309 5.1: Multi UA', (it) => { + const robotsTxt = new RobotsTxt(rfcOneRobots); + + const ua1 = 'barbot'; + const ua2 = 'bazbot'; + + it.is(robotsTxt.isPathAllowed('/', ua1), true); + it.is(robotsTxt.isPathAllowed('/foo', ua1), true); + it.is(robotsTxt.isPathAllowed('/foo/bar', ua1), true); + it.is(robotsTxt.isPathAllowed('/example/page.html', ua1), false); + it.is(robotsTxt.isPathAllowed('/example/page.html/test', ua1), false); + it.is(robotsTxt.isPathAllowed('/example/allowed.gif', ua1), true); + it.is(robotsTxt.isPathAllowed('/example/allowed.gif/xyz', ua1), true); + it.is(robotsTxt.isPathAllowed('/example/', ua1), true); + it.is(robotsTxt.isPathAllowed('/publications/', ua1), true); + + it.is(robotsTxt.isPathAllowed('/', ua2), true); + it.is(robotsTxt.isPathAllowed('/foo', ua2), true); + it.is(robotsTxt.isPathAllowed('/foo/bar', ua2), true); + it.is(robotsTxt.isPathAllowed('/example/page.html', ua2), false); + it.is(robotsTxt.isPathAllowed('/example/page.html/test', ua2), false); + it.is(robotsTxt.isPathAllowed('/example/allowed.gif', ua2), true); + it.is(robotsTxt.isPathAllowed('/example/allowed.gif/xyz', ua2), true); + it.is(robotsTxt.isPathAllowed('/example/', ua2), true); + it.is(robotsTxt.isPathAllowed('/publications/', ua2), true); +}); + +test('RFC 9309 5.1: Empty UA field', (it) => { + const robotsTxt = new RobotsTxt(rfcOneRobots); + + const ua = 'quxbot'; + + it.is(robotsTxt.isPathAllowed('/', ua), true); + it.is(robotsTxt.isPathAllowed('/foo', ua), true); + it.is(robotsTxt.isPathAllowed('/foo/bar', ua), true); + it.is(robotsTxt.isPathAllowed('/foo/bin.gif', ua), true); + it.is(robotsTxt.isPathAllowed('/foo/bin.gif/baz', ua), true); + it.is(robotsTxt.isPathAllowed('/example', ua), true); + it.is(robotsTxt.isPathAllowed('/example/', ua), true); + it.is(robotsTxt.isPathAllowed('/example/foo', ua), true); + it.is(robotsTxt.isPathAllowed('/publications/', ua), true); +}); + +test('RFC 9309 5.2: Path length precedence', (it) => { + const robotsTxt = new RobotsTxt(rfcTwoRobots); + + const ua = 'foobot'; + + it.is(robotsTxt.isPathAllowed('/example/page/', ua), true); + it.is(robotsTxt.isPathAllowed('/example/page/disallowed.gif', ua), false); +}); + +test('RFC 9309 2.2.2: Allow duplicate precedence', (it) => { + const robotsTxt = new RobotsTxt(rfcThreeRobots); + + const ua = 'foobot'; + + it.is(robotsTxt.isPathAllowed('/example/page/', ua), true); +}); + +test('RFC 9309 2.2.2: Allow duplicate precedence (reversed)', (it) => { + const robotsTxt = new RobotsTxt(rfcFourRobots); + + const ua = 'foobot'; + + it.is(robotsTxt.isPathAllowed('/example/page/', ua), true); +}); From bf2cee7be82b310bbdbb88256d2cb17b20cb4763 Mon Sep 17 00:00:00 2001 From: Russell Steadman Date: Thu, 12 Oct 2023 01:41:11 +0000 Subject: [PATCH 4/4] Memory and redirect RFC updates to netscrape --- packages/bot/src/errors.ts | 5 +++++ packages/bot/src/index.ts | 6 ++++++ packages/bot/src/test/_server.util.ts | 2 ++ packages/bot/src/test/bot.spec.ts | 31 ++++++++++----------------- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/packages/bot/src/errors.ts b/packages/bot/src/errors.ts index b135e86..498ddb9 100644 --- a/packages/bot/src/errors.ts +++ b/packages/bot/src/errors.ts @@ -3,6 +3,7 @@ enum BotErrorType { Delay, RobotsTxt, Configuration, + MemorySafety, } export class BotError extends Error implements Error { @@ -21,3 +22,7 @@ export class DelayError extends BotError implements BotError { export class ConfigError extends BotError implements BotError { type = BotErrorType.Configuration; } + +export class MemoryError extends BotError implements BotError { + type = BotErrorType.MemorySafety; +} diff --git a/packages/bot/src/index.ts b/packages/bot/src/index.ts index 21a53bd..dede70b 100644 --- a/packages/bot/src/index.ts +++ b/packages/bot/src/index.ts @@ -178,6 +178,7 @@ class Bot { shared: false, immutableMinTimeToLive: 3600 * 1000, }, + maxRedirects: 5, dnsCache: this.options.disableCaching ? false : this.dnsCachable, ...options?.overrides, }; @@ -211,6 +212,11 @@ class Bot { throw new Errors.RobotsRejection('Robots.txt server error'); } + if (Buffer.byteLength(robotsTxt.body) > 5e5) { + // RFC 9309 2.5: Can reject robots.txt files larger than 500KB + throw new Errors.MemoryError('Robots.txt too large'); + } + // Parse the robots.txt this.robotsTxt[origin] = new RobotsTxt(robotsTxt.body); diff --git a/packages/bot/src/test/_server.util.ts b/packages/bot/src/test/_server.util.ts index 4a4e512..6f24be8 100644 --- a/packages/bot/src/test/_server.util.ts +++ b/packages/bot/src/test/_server.util.ts @@ -51,6 +51,8 @@ export async function startServer( }); }); + server.setMaxListeners(0); + return { server, requests, port }; } diff --git a/packages/bot/src/test/bot.spec.ts b/packages/bot/src/test/bot.spec.ts index 30c8ae7..6036b7f 100644 --- a/packages/bot/src/test/bot.spec.ts +++ b/packages/bot/src/test/bot.spec.ts @@ -1,11 +1,8 @@ import test from 'ava'; import Bot from '../index.js'; import * as Errors from '../errors.js'; -import { type Server } from 'http'; import { startServer, stopServer } from './_server.util.js'; -let server: Server | undefined; - test('Bot#constructor()', (it) => { const bot = new Bot({ name: 'Test', version: '0.1' }); it.is(typeof bot.makeRequest, 'function'); @@ -13,7 +10,7 @@ test('Bot#constructor()', (it) => { test('Bot disallow none', async (it) => { const start = await startServer('User-agent: *\nDisallow:'); - server = start.server; + it.teardown(async () => await stopServer(start.server)); const bot = new Bot({ name: 'Test', version: '0.1' }); @@ -51,7 +48,7 @@ test('Bot disallow none', async (it) => { test('Bot allow all', async (it) => { const start = await startServer('User-agent: *\nAllow: /'); - server = start.server; + it.teardown(async () => await stopServer(start.server)); const bot = new Bot({ name: 'Test', version: '0.1' }); @@ -70,7 +67,7 @@ test('Bot allow all', async (it) => { test('Bot disallow all', async (it) => { const start = await startServer('User-agent: *\nDisallow: /'); - server = start.server; + it.teardown(async () => await stopServer(start.server)); const bot = new Bot({ name: 'Test', version: '0.1' }); @@ -107,11 +104,11 @@ test('Bot disallow all', async (it) => { ); }); -test('Bot responds to 400 errors', async (it) => { +test('RFC 9309 2.3.1.3: Responds to 400 errors', async (it) => { const start = await startServer('User-agent: *\nAllow: /', { robots400: true, }); - server = start.server; + it.teardown(async () => await stopServer(start.server)); const bot = new Bot({ name: 'Test', version: '0.1' }); @@ -128,11 +125,11 @@ test('Bot responds to 400 errors', async (it) => { it.is(req.body, 'C'); }); -test('Bot responds to 500 errors', async (it) => { +test('RFC 9309 2.3.1.4: Responds to 500 errors', async (it) => { const start = await startServer('User-agent: *\nAllow: /', { robots500: true, }); - server = start.server; + it.teardown(async () => await stopServer(start.server)); const bot = new Bot({ name: 'Test', version: '0.1' }); @@ -140,7 +137,7 @@ test('Bot responds to 500 errors', async (it) => { () => bot.makeRequest(`http://127.0.0.1:${start.port}/`), { instanceOf: Errors.RobotsRejection, - message: 'Request blocked by robots.txt', + message: 'Robots.txt server error', }, ); @@ -148,7 +145,7 @@ test('Bot responds to 500 errors', async (it) => { () => bot.makeRequest(`http://127.0.0.1:${start.port}/a`), { instanceOf: Errors.RobotsRejection, - message: 'Request blocked by robots.txt', + message: 'Robots.txt server error', }, ); @@ -156,7 +153,7 @@ test('Bot responds to 500 errors', async (it) => { () => bot.makeRequest(`http://127.0.0.1:${start.port}/a/b`), { instanceOf: Errors.RobotsRejection, - message: 'Request blocked by robots.txt', + message: 'Robots.txt server error', }, ); @@ -164,13 +161,7 @@ test('Bot responds to 500 errors', async (it) => { () => bot.makeRequest(`http://127.0.0.1:${start.port}/a/b/c`), { instanceOf: Errors.RobotsRejection, - message: 'Request blocked by robots.txt', + message: 'Robots.txt server error', }, ); }); - -test.afterEach(async () => { - if (server) { - await stopServer(server); - } -});