From 0667ee023dace52944521b57425749d8a22f3814 Mon Sep 17 00:00:00 2001 From: RuiSiang <49300088+RuiSiang@users.noreply.github.com> Date: Tue, 23 Mar 2021 11:32:04 +0800 Subject: [PATCH 1/3] refactor pow and restructure files --- app.ts | 2 +- lib/bundle.js | 1 - lib/pow.ts | 10 - package-lock.json | 9 +- package.json | 2 - routes/pow-router.ts | 2 +- scripts/build.sh | 4 +- service/bundle.js | 1 + service/controllers/blacklist.ts | 5 +- service/controllers/rate-limiter.ts | 4 +- service/controllers/waf.ts | 146 +++++ service/pow-service.ts | 2 +- {lib => service}/pow/solver.ts | 21 +- {lib => service}/pow/utils.ts | 22 +- {lib => service}/pow/verifier.ts | 22 +- service/{ => util}/database-service.ts | 0 tests/unit/database.test.ts | 2 +- tests/unit/pow.test.ts | 2 +- wafRules.json | 722 +++++++++++++++++++++++++ wafTypes.json | 25 + 20 files changed, 942 insertions(+), 62 deletions(-) delete mode 100644 lib/bundle.js delete mode 100644 lib/pow.ts create mode 100644 service/bundle.js create mode 100644 service/controllers/waf.ts rename {lib => service}/pow/solver.ts (59%) rename {lib => service}/pow/utils.ts (67%) rename {lib => service}/pow/verifier.ts (55%) rename service/{ => util}/database-service.ts (100%) create mode 100644 wafRules.json create mode 100644 wafTypes.json diff --git a/app.ts b/app.ts index a7f11dc9..67be5e6f 100644 --- a/app.ts +++ b/app.ts @@ -8,7 +8,7 @@ import { createProxyMiddleware } from 'http-proxy-middleware' import c2k from 'koa2-connect' import session from 'koa-session-minimal' -import config from './service/config-parser' +import config from './service/util/config-parser' import powRouter from './routes/pow-router' import testRouter from './routes/test-router' import { controller } from './service/controller-service' diff --git a/lib/bundle.js b/lib/bundle.js deleted file mode 100644 index beae2575..00000000 --- a/lib/bundle.js +++ /dev/null @@ -1 +0,0 @@ -window.powSolver = require('../dist/lib/pow.js').Solver; \ No newline at end of file diff --git a/lib/pow.ts b/lib/pow.ts deleted file mode 100644 index e49d84f4..00000000 --- a/lib/pow.ts +++ /dev/null @@ -1,10 +0,0 @@ -/* ****************************************************************************************** */ -/* This module is a improved and typescript version of proof-of-work package by Fedor Indutny */ -/* The original module can be found at https://github.com/indutny/proof-of-work for reference */ -/* ****************************************************************************************** */ - -import utils from './pow/utils' -import Solver from './pow/solver' -import Verifier from './pow/verifier' - -export { utils, Solver, Verifier } diff --git a/package-lock.json b/package-lock.json index 3555aa73..6b734601 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1330,12 +1330,6 @@ "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.2.tgz", "integrity": "sha512-YATxVxgRqNH6nHEIsvg6k2Boc1JHI9ZbH5iWFFv/MTkchz3b1ieGDa5T0a9RznNdI0KhVbdbWSN+KWWrQZRxTw==" }, - "@types/minimalistic-assert": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@types/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-LjFxDeQtgj+MYOOFuV+BILc/w9ryfxd2eNLF0oJdOBud+xCj80ZAEkAwt6y0qx81QMZ632yAaWdvIMjEK2HcmA==", - "dev": true - }, "@types/node": { "version": "14.14.32", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.32.tgz", @@ -7129,7 +7123,8 @@ "minimalistic-assert": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==", + "dev": true }, "minimalistic-crypto-utils": { "version": "1.0.1", diff --git a/package.json b/package.json index 3a30f69f..355cb0dd 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,6 @@ "koa-static": "^5.0.0", "koa-views": "^7.0.1", "koa2-connect": "^1.0.2", - "minimalistic-assert": "^1.0.1", "moment": "^2.29.1", "pug": "^3.0.1", "randomstring": "^1.1.5", @@ -42,7 +41,6 @@ "@types/koa-router": "^7.4.1", "@types/koa-session-minimal": "^3.0.5", "@types/koa-views": "^2.0.4", - "@types/minimalistic-assert": "^1.0.1", "@types/randomstring": "^1.1.6", "@types/sqlite3": "^3.1.7", "awesome-typescript-loader": "^5.2.1", diff --git a/routes/pow-router.ts b/routes/pow-router.ts index 491cb830..be72d5ce 100644 --- a/routes/pow-router.ts +++ b/routes/pow-router.ts @@ -2,7 +2,7 @@ import Koa from 'koa' import Router from 'koa-router' import Pow from '../service/pow-service' -import config from '../service/config-parser' +import config from '../service/util/config-parser' const router = new Router() const pow = new Pow(config.initial_difficulty) diff --git a/scripts/build.sh b/scripts/build.sh index 9c11f384..f399def9 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -6,7 +6,9 @@ echo "Build Successful!" && \ echo "Copying Files..." && \ cp -rfRT views dist/views && \ cp -rfRT public dist/public && \ +cp wafRules.json dist/wafRules.json && \ +cp wafTypes.json dist/wafTypes.json && \ cp package.json dist/package.json && \ cp .env dist/.env && \ -browserify lib/bundle.js -t uglifyify | uglifyjs > dist/public/javascripts/bundle.min.js && \ +browserify service/bundle.js -t uglifyify | uglifyjs > dist/public/javascripts/bundle.min.js && \ echo "Success!" \ No newline at end of file diff --git a/service/bundle.js b/service/bundle.js new file mode 100644 index 00000000..b9272336 --- /dev/null +++ b/service/bundle.js @@ -0,0 +1 @@ +window.powSolver = require('../dist/service/pow/solver.js').Solver; \ No newline at end of file diff --git a/service/controllers/blacklist.ts b/service/controllers/blacklist.ts index 1dc65116..711aae10 100644 --- a/service/controllers/blacklist.ts +++ b/service/controllers/blacklist.ts @@ -1,5 +1,6 @@ import { CronJob } from 'cron' -import Database from '../database-service' +import Database from '../util/database-service' +import config from '../util/config-parser' class Blacklist { private static instance: Blacklist @@ -22,7 +23,7 @@ class Blacklist { if (!dbQuery.length) { return true } - return false + return false || !config.rate_limit } public ban = async (ip: string, minutes: number) => { await this.db.queryAsync({ diff --git a/service/controllers/rate-limiter.ts b/service/controllers/rate-limiter.ts index 5674a8d4..7ee05495 100644 --- a/service/controllers/rate-limiter.ts +++ b/service/controllers/rate-limiter.ts @@ -1,8 +1,8 @@ import moment from 'moment' import { CronJob } from 'cron' -import Database from '../database-service' +import Database from '../util/database-service' import Blacklist from './blacklist' -import config from '../config-parser' +import config from '../util/config-parser' class RateLimiter { private static instance: RateLimiter diff --git a/service/controllers/waf.ts b/service/controllers/waf.ts new file mode 100644 index 00000000..d0b62003 --- /dev/null +++ b/service/controllers/waf.ts @@ -0,0 +1,146 @@ +import fs from 'fs' +import path from 'path' +import config from '../util/config-parser' +import { ParameterizedContext } from 'koa' + +interface Rule { + reg: RegExp + type: number + cmt: string +} + +interface _Rule { + id: number + reg: string + type: number + cmt: string +} + +class Waf { + private static instance: Waf + public static getInstance(): Waf { + if (!Waf.instance) { + Waf.instance = new Waf() + } + return Waf.instance + } + + constructor() { + this.load() + } + + private types: { + [key: string]: string + } = {} + + private rules: { + [key: string]: Rule + } = {} + + private entries: { [key: string]: RegExp } = {} + + private load = () => { + this.types = JSON.parse( + fs.readFileSync(path.join(process.cwd(), 'wafTypes.json')).toString() + ) + const rulesJson = JSON.parse( + fs.readFileSync(path.join(process.cwd(), 'wafRules.json')).toString() + ) + + for (let key in this.types) { + this.types[this.types[key]] = key + } + rulesJson.forEach((rule: _Rule) => { + this.rules[rule.id] = { + reg: new RegExp(rule.reg), + type: rule.type, + cmt: rule.cmt, + } + this.entries[rule.id] = new RegExp(rule.reg) + }) + } + + private parseNumString = (numString: string) => { + const substrings = numString.split(',') + let numArr: number[] = [] + substrings.forEach(function (item) { + if (!!item) { + const tmpArr = item.split('-') + if (tmpArr.length == 2) { + const low = parseInt(tmpArr[0]) + const high = parseInt(tmpArr[1]) + for (let i = low; i <= high; i++) { + numArr.push(i) + } + } else { + numArr.push(parseInt(tmpArr[0])) + } + } + }) + return numArr + } + + private detect = async (test: string, excludes: number[]) => { + for (let key in this.entries) { + if (!excludes.includes(parseInt(key))) { + if (this.entries[key].test(test) === true) { + return key + } + } + } + return 0 + } + + public scan = async (ctx: ParameterizedContext) => { + if (!!config.waf) { + const urlExcludeRules = this.parseNumString(config.waf_url_exclude_rules) + const urlResult = await this.detect(ctx.url, urlExcludeRules) + if (!!urlResult) { + return { + id: urlResult, + type: this.types[this.rules[urlResult].type], + cmt: this.rules[urlResult].cmt, + location: 'url', + } + } + const headerExcludeRules = this.parseNumString( + config.waf_header_exclude_rules + ) + const headerResult = await this.detect( + JSON.stringify(ctx.headers), + headerExcludeRules + ) + if (!!headerResult) { + return { + id: headerResult, + type: this.types[this.rules[headerResult].type], + cmt: this.rules[headerResult].cmt, + location: 'header', + } + } + const bodyExcludeRules = this.parseNumString( + config.waf_body_exclude_rules + ) + const bodyResult = await this.detect( + JSON.stringify(ctx.request.body), + bodyExcludeRules + ) + if (!!bodyResult) { + return { + id: bodyResult, + type: this.types[this.rules[bodyResult].type], + cmt: this.rules[bodyResult].cmt, + location: 'body', + } + } + } + return null + } + public test = async (test: string, excludes: number[]) => { + if (process.env.NODE_ENV === 'test') { + return await this.detect(test, excludes) + } + return 0 + } +} +export default Waf diff --git a/service/pow-service.ts b/service/pow-service.ts index 0f96905a..ffe33299 100644 --- a/service/pow-service.ts +++ b/service/pow-service.ts @@ -1,5 +1,5 @@ import randomstring from 'randomstring' -import { Verifier } from '../lib/pow' +import Verifier from './pow/verifier' class Pow { constructor(initDifficulty: number) { diff --git a/lib/pow/solver.ts b/service/pow/solver.ts similarity index 59% rename from lib/pow/solver.ts rename to service/pow/solver.ts index 6b4c9f71..aa04b2e8 100644 --- a/lib/pow/solver.ts +++ b/service/pow/solver.ts @@ -1,30 +1,33 @@ import utils from './utils' -const MIN_NONCE_SIZE = 8 -const NONCE_SIZE = MIN_NONCE_SIZE + 8 +const minNonceSize = 8 +const nonceSize = minNonceSize + 8 class Solver { public solve = async ( complexity: number, prefix: string ): Promise => { - let nonce = await utils.allocBuffer(NONCE_SIZE) - for (;;) { - await this._genNonce(nonce) + let nonce = Buffer.alloc(nonceSize) + while (true) { + await this.genNonce(nonce) const hash = await utils.hash(nonce, prefix) if (await utils.checkComplexity(hash, complexity)) { return nonce } } } - private _genNonce = async (buf) => { + private genNonce = async (buf: Buffer) => { const now = Date.now() let off = await utils.writeTimestamp(buf, now, 0) const words = off + (((buf.length - off) / 4) | 0) * 4 - for (; off < words; off += 4) + for (; off < words; off += 4) { utils.writeUInt32(buf, (Math.random() * 0x100000000) >>> 0, off) - for (; off < buf.length; off++) buf[off] = (Math.random() * 0x100) >>> 0 + } + for (; off < buf.length; off++) { + buf[off] = (Math.random() * 0x100) >>> 0 + } } } -export default Solver +export { Solver } diff --git a/lib/pow/utils.ts b/service/pow/utils.ts similarity index 67% rename from lib/pow/utils.ts rename to service/pow/utils.ts index 084b0c45..43756f68 100644 --- a/lib/pow/utils.ts +++ b/service/pow/utils.ts @@ -1,10 +1,4 @@ import createHash from 'create-hash' -import assert from 'minimalistic-assert' -const EMPTY_BUFFER = Buffer.alloc(0) - -const allocBuffer = async (size: number) => { - return Buffer.alloc(size) -} const writeUInt32 = async (buffer: any, value: number, off: number) => { buffer.writeUInt32LE(value, off, true) @@ -12,10 +6,10 @@ const writeUInt32 = async (buffer: any, value: number, off: number) => { } const writeTimestamp = async (buffer: any, ts: number, off: number) => { - const hi = (ts / 0x100000000) >>> 0 - const lo = (ts & 0xffffffff) >>> 0 - buffer.writeUInt32BE(hi, off, true) - buffer.writeUInt32BE(lo, off + 4, true) + const high = (ts / 0x100000000) >>> 0 + const low = (ts & 0xffffffff) >>> 0 + buffer.writeUInt32BE(high, off, true) + buffer.writeUInt32BE(low, off + 4, true) return off + 8 } @@ -25,13 +19,15 @@ const readTimestamp = async (buffer: Buffer, off: number) => { const hash = async (nonce: Buffer, prefix: string) => { const h = createHash('sha256') - if (prefix) h.update(prefix,'hex') + if (prefix) h.update(prefix, 'hex') h.update(nonce) return h.digest() } const checkComplexity = async (hash: Buffer, complexity: number) => { - assert(complexity < hash.length * 8, 'Complexity is too high') + if (complexity >= hash.length * 8) { + throw 'Complexity is too high' + } let off = 0 let i: number for (i = 0; i <= complexity - 8; i += 8, off++) { @@ -45,9 +41,7 @@ const checkComplexity = async (hash: Buffer, complexity: number) => { export default { writeTimestamp, writeUInt32, - allocBuffer, hash, checkComplexity, readTimestamp, - EMPTY_BUFFER, } diff --git a/lib/pow/verifier.ts b/service/pow/verifier.ts similarity index 55% rename from lib/pow/verifier.ts rename to service/pow/verifier.ts index 74c7a1ae..857b6ac4 100644 --- a/lib/pow/verifier.ts +++ b/service/pow/verifier.ts @@ -1,8 +1,8 @@ -import config from '../../service/config-parser' +import config from '../util/config-parser' import utils from './utils' -const MIN_NONCE_LEN = 8 -const MAX_NONCE_LEN = 32 +const minNonceSize = 8 +const maxNonceSize = 32 const DEFAULT_VALIDITY = config.nonce_validity export interface IVerifierOptions { @@ -21,13 +21,17 @@ export class Verifier { complexity: number, prefix: string ): Promise => { - if (nonce.length < MIN_NONCE_LEN) return false - if (nonce.length > MAX_NONCE_LEN) return false - const ts = await utils.readTimestamp(nonce, 0) - const now = Date.now() - if (Math.abs(ts - now) > this.validity) return false + if (nonce.length < minNonceSize || nonce.length > maxNonceSize) { + return false + } + const diff = (await utils.readTimestamp(nonce, 0)) - Date.now() + if (Math.abs(diff) > this.validity) { + return false + } const hash = await utils.hash(nonce, prefix) - if (!(await utils.checkComplexity(hash, complexity))) return false + if (!(await utils.checkComplexity(hash, complexity))) { + return false + } return true } } diff --git a/service/database-service.ts b/service/util/database-service.ts similarity index 100% rename from service/database-service.ts rename to service/util/database-service.ts diff --git a/tests/unit/database.test.ts b/tests/unit/database.test.ts index 10bd247d..4318529e 100644 --- a/tests/unit/database.test.ts +++ b/tests/unit/database.test.ts @@ -1,4 +1,4 @@ -import Database from '../../service/database-service' +import Database from '../../service/util/database-service' let db: Database const tables = [ diff --git a/tests/unit/pow.test.ts b/tests/unit/pow.test.ts index 59a3888c..0c841da7 100644 --- a/tests/unit/pow.test.ts +++ b/tests/unit/pow.test.ts @@ -1,5 +1,5 @@ import Pow from '../../service/pow-service' -import { Solver } from '../../lib/pow' +import { Solver } from '../../service/pow/solver' let pow: Pow let solver: Solver diff --git a/wafRules.json b/wafRules.json new file mode 100644 index 00000000..49f7a568 --- /dev/null +++ b/wafRules.json @@ -0,0 +1,722 @@ +[ + { + "id": 1, + "reg": "\\(\\)\\s*\\{.*?;\\s*\\}\\s*;", + "type": 9, + "cmt": "Shellshock (CVE-2014-6271)" + }, + { + "id": 2, + "reg": "\\(\\)\\s*\\{.*?\\(.*?\\).*?=>.*?\\\\'", + "type": 9, + "cmt": "Shellshock (CVE-2014-7169)" + }, + { + "id": 3, + "reg": "\\{\\{.*?\\}\\}", + "type": 4, + "cmt": "Flask curly syntax" + }, + { + "id": 4, + "reg": "\\bfind_in_set\\b.*?\\(.+?,.+?\\)", + "type": 6, + "cmt": "Common MySQL function \"find_in_set\"" + }, + { + "id": 5, + "reg": "[\"'].*?>", + "type": 3, + "cmt": "HTML breaking" + }, + { + "id": 6, + "reg": "\\bsqlite_master\\b", + "type": 7, + "cmt": "SQLite information disclosure \"sqlite_master\"" + }, + { + "id": 7, + "reg": "\\bmysql.*?\\..*?user\\b", + "type": 7, + "cmt": "MySQL information disclosure \"mysql.user\"" + }, + { + "id": 8, + "reg": "#.+?\\)[\"\\s]*>", + "type": 5, + "cmt": "HTML breaking" + }, + { + "id": 9, + "reg": "['\"][,;\\s]+\\w*[\\[\\(]", + "type": 3, + "cmt": "HTML breaking" + }, + { + "id": 10, + "reg": ">.*?<\\s*\\/?[\\w\\s]+>", + "type": 3, + "cmt": "Unquoted HTML breaking with closing tag" + }, + { + "id": 11, + "reg": "\\blocation\\b.*?\\..*?\\bhash\\b", + "type": 2, + "cmt": "JavaScript \"location.hash\"" + }, + { + "id": 12, + "reg": "\\bwith\\b\\s*\\(.+?\\)[\\s\\w]+\\(", + "type": 6, + "cmt": "Self-contained payload" + }, + { + "id": 13, + "reg": "(\\b(do|while|for)\\b.*?\\([^)]*\\).*?\\{)|(\\}.*?\\b(do|while|for)\\b.*?\\([^)]*\\))", + "type": 4, + "cmt": "C-style loops" + }, + { + "id": 14, + "reg": "[=(].+?\\?.+?:", + "type": 2, + "cmt": "C-style ternary operator" + }, + { + "id": 15, + "reg": "\\\\u00[a-f0-9]{2}", + "type": 1, + "cmt": "Octal entity" + }, + { + "id": 16, + "reg": "\\\\x0*[a-f0-9]{2}", + "type": 1, + "cmt": "Hex entity" + }, + { + "id": 17, + "reg": "\\\\\\d{2,3}", + "type": 1, + "cmt": "Unicode entity" + }, + { + "id": 18, + "reg": "\\.\\.[\\/\\\\]", + "type": 4, + "cmt": "Directory traversal" + }, + { + "id": 19, + "reg": "%(c0\\.|af\\.|5c\\.)", + "type": 4, + "cmt": "Directory traversal unicode + urlencoding" + }, + { + "id": 20, + "reg": "%2e%2e[\\/\\\\]", + "type": 4, + "cmt": "Directory traversal urlencoding" + }, + { + "id": 21, + "reg": "%c0%ae[\\/\\\\]", + "type": 4, + "cmt": "Directory traversal unicode + urlencoding" + }, + { + "id": 22, + "reg": "\\.(ht(access|passwd|group))|(apache|httpd)\\d?\\.conf", + "type": 4, + "cmt": "Common Apache files" + }, + { + "id": 23, + "reg": "\\/etc\\/[.\\/]*(passwd|shadow|master\\.passwd)", + "type": 4, + "cmt": "Common Unix files" + }, + { + "id": 24, + "reg": "\\bdata:.*?,", + "type": 2, + "cmt": "Data URI scheme" + }, + { + "id": 25, + "reg": ";base64|base64,", + "type": 2, + "cmt": "Data URI scheme \"base64\"" + }, + { + "id": 26, + "reg": "php:\\/\\/filter", + "type": 6, + "cmt": "PHP input/output stream filter" + }, + { + "id": 27, + "reg": "php:\\/\\/input", + "type": 6, + "cmt": "PHP input stream" + }, + { + "id": 28, + "reg": "php:\\/\\/output", + "type": 6, + "cmt": "PHP output stream" + }, + { + "id": 29, + "reg": "convert\\.base64-(de|en)code", + "type": 6, + "cmt": "PHP input/output stream filter \"base64\"" + }, + { + "id": 30, + "reg": "zlib\\.(de|in)flate", + "type": 6, + "cmt": "PHP input/output stream filter \"zlib\"" + }, + { + "id": 31, + "reg": "@import\\b", + "type": 3, + "cmt": "CSS \"import\"" + }, + { + "id": 32, + "reg": "\\burl\\s*\\(.+?\\)", + "type": 2, + "cmt": "CSS pointer to resource" + }, + { + "id": 33, + "reg": "\\/\\/.+?\\/", + "type": 1, + "cmt": "URL" + }, + { + "id": 34, + "reg": "\\)\\s*\\[", + "type": 2, + "cmt": "JavaScript language construct" + }, + { + "id": 35, + "reg": "<\\?(?!xml\\s)", + "type": 3, + "cmt": "PHP opening tag" + }, + { + "id": 36, + "reg": "%(HOME(DRIVE|PATH)|SYSTEM(DRIVE|ROOT)|WINDIR|USER(DOMAIN|PROFILE|NAME)|((LOCAL)?APP|PROGRAM)DATA)%", + "type": 2, + "cmt": "Common Windows environment variable" + }, + { + "id": 37, + "reg": "%\\w+%", + "type": 2, + "cmt": "Windows environment variable pattern" + }, + { + "id": 38, + "reg": "\\bunion\\b.+?\\bselect\\b", + "type": 3, + "cmt": "Common SQL command \"union select\"" + }, + { + "id": 39, + "reg": "\\bupdate\\b.+?\\bset\\b", + "type": 3, + "cmt": "Common SQL command \"update\"" + }, + { + "id": 40, + "reg": "\\bdrop\\b.+?\\b(database|table)\\b", + "type": 3, + "cmt": "Common SQL command \"drop\"" + }, + { + "id": 41, + "reg": "\\bdelete\\b.+?\\bfrom\\b", + "type": 3, + "cmt": "Common SQL command \"delete\"" + }, + { + "id": 42, + "reg": "--.+?", + "type": 1, + "cmt": "Common SQL comment syntax" + }, + { + "id": 43, + "reg": "\\[\\$(ne|eq|lte?|gte?|n?in|mod|all|size|exists|type|slice|or)\\]", + "type": 5, + "cmt": "MongoDB SQL commands" + }, + { + "id": 44, + "reg": "\\$\\(.+?\\)", + "type": 2, + "cmt": "jQuery selector" + }, + { + "id": 45, + "reg": "\\/\\*.*?\\*\\/", + "type": 3, + "cmt": "C-style comment syntax" + }, + { + "id": 46, + "reg": "", + "type": 3, + "cmt": "XML comment syntax" + }, + { + "id": 47, + "reg": "", + "type": 6, + "cmt": "Base URL" + }, + { + "id": 48, + "reg": "])", + "type": 2, + "cmt": "Common SQL comparison \"where\"" + }, + { + "id": 109, + "reg": "\\bif\\b.*?\\(.+?,.+?,.+?\\)", + "type": 2, + "cmt": "Common SQL comparison \"if\"" + }, + { + "id": 110, + "reg": "\\b(ifnull|nullif)\\b.*?\\(.+?,.+?\\)", + "type": 3, + "cmt": "Common SQL comparison \"ifnull\"" + }, + { + "id": 111, + "reg": "\\bwhere\\b.+?(\\b(n?and|x?or|not)\\b|(\\&\\&|\\|\\|))", + "type": 3, + "cmt": "Common SQL comparison \"where\"" + }, + { + "id": 112, + "reg": "\\bcase\\b.+?\\bwhen\\b.+?\\bend\\b", + "type": 4, + "cmt": "Common SQL comparison \"case\"" + }, + { + "id": 113, + "reg": "\\bexec\\b.+?\\bxp_cmdshell\\b", + "type": 9, + "cmt": "MSSQL code execution \"xp_cmdshell\"" + }, + { + "id": 114, + "reg": "\\bcreate\\b.+?\\b(procedure|function)\\b.*?\\(.*?\\)", + "type": 4, + "cmt": "Common SQL command \"create\"" + }, + { + "id": 115, + "reg": "\\binsert\\b.+?\\binto\\b.*?\\bvalues\\b.*?\\(.+?\\)", + "type": 5, + "cmt": "Common SQL command \"insert\"" + }, + { + "id": 116, + "reg": "\\bselect\\b.+?\\bfrom\\b", + "type": 3, + "cmt": "Common SQL command \"select\"" + }, + { + "id": 117, + "reg": "\\bpg_user\\b", + "type": 7, + "cmt": "PgSQL information disclosure \"pg_user\"" + }, + { + "id": 118, + "reg": "\\bpg_database\\b", + "type": 7, + "cmt": "PgSQL information disclosure \"pg_database\"" + }, + { + "id": 119, + "reg": "\\bpg_shadow\\b", + "type": 7, + "cmt": "PgSQL information disclosure \"pg_shadow\"" + }, + { + "id": 120, + "reg": "\\b(current_)?database\\b.*?\\(.*?\\)", + "type": 2, + "cmt": "Common SQL function \"database\"" + } +] diff --git a/wafTypes.json b/wafTypes.json new file mode 100644 index 00000000..c541c25c --- /dev/null +++ b/wafTypes.json @@ -0,0 +1,25 @@ +{ + "1": "xss", + "2": "win", + "3": "unix", + "4": "rce", + "5": "lfi", + "6": "rfi", + "7": "sqli", + "8": "spam", + "9": "dos", + "10": "php", + "11": "perl", + "12": "python", + "13": "xxe", + "14": "ldap", + "15": "bash", + "16": "id", + "17": "mysql", + "18": "pgsql", + "19": "sqlite", + "20": "mongo", + "21": "tsql", + "22": "mssql", + "23": "css" +} From 67e7f56758c2a80211a2caf1607d9a16ebd01889 Mon Sep 17 00:00:00 2001 From: RuiSiang <49300088+RuiSiang@users.noreply.github.com> Date: Tue, 23 Mar 2021 11:32:22 +0800 Subject: [PATCH 2/3] add waf functionalities --- .env.example | 8 +++-- CONFIGURE.md | 5 ++- README.md | 6 +++- public/stylesheets/style.css | 1 + service/controller-service.ts | 46 ++++++++++++++++++---------- service/{ => util}/config-parser.ts | 15 +++++++-- tests/integration/controller.test.ts | 18 +++++++++-- tests/unit/config.spec.ts | 7 +++-- tests/unit/waf.test.ts | 19 ++++++++++++ views/waf.pug | 13 ++++++++ 10 files changed, 110 insertions(+), 28 deletions(-) rename service/{ => util}/config-parser.ts (80%) create mode 100644 tests/unit/waf.test.ts create mode 100644 views/waf.pug diff --git a/.env.example b/.env.example index 6f73bf52..3c003fa5 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,5 @@ PORT=3000 SESSION_KEY="abcdefghijklmnop" -WAF=on POW=on NONCE_VALIDITY=60000 INITIAL_DIFFICULTY=13 @@ -11,4 +10,9 @@ RATE_LIMIT_SAMPLE_MINUTES=60 RATE_LIMIT_SESSION_THRESHOLD=100 RATE_LIMIT_BAN_IP=on RATE_LIMIT_IP_THRESHOLD=500 -RATE_LIMIT_BAN_MINUTES=15 \ No newline at end of file +RATE_LIMIT_BAN_MINUTES=15 + +WAF=on +WAF_URL_EXCLUDE_RULES= +WAF_HEADER_EXCLUDE_RULES=14,33,80,96,100 +WAF_BODY_EXCLUDE_RULES= diff --git a/CONFIGURE.md b/CONFIGURE.md index 39c17970..3bcb2991 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -31,4 +31,7 @@ Ratelimit Options WAF Options -- WAF: (default:on) toggles waf functionality on/off (waf is still a work in progress) +- WAF: (default:on) toggles waf functionality on/off +- WAF_URL_EXCLUDE_RULES: exclude rules to check when scanning request url, use ',' to seperate rule numbers, use '-' to specify a range (eg: 1,2-4,5,7-10) +- WAF_HEADER_EXCLUDE_RULES: (default:14,33,80,96,100) exclude rules to check when scanning request header, use ',' to seperate rule numbers, use '-' to specify a range (eg: 1,2-4,5,7-10) +- WAF_BODY_EXCLUDE_RULES: exclude rules to check when scanning request body, use ',' to seperate rule numbers, use '-' to specify a range (eg: 1,2-4,5,7-10) diff --git a/README.md b/README.md index 899a5256..d69c497e 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,10 @@ nodejs and docker envitonment variables +## References ++ Proof-of-work by Fedor Indutny (PoW utility functions) ++ Shadowd by Zesecure (WAF rules) + ## TODOs - [x] Web Service Structure @@ -48,7 +52,7 @@ envitonment variables - [x] IP Blacklisting - [x] Ratelimiting - [x] Unit Testing -- [ ] WAF Implementation +- [x] WAF Implementation - [ ] Dynamic Difficulty - [ ] Multi-Instance Syncing - [ ] Monitoring diff --git a/public/stylesheets/style.css b/public/stylesheets/style.css index d09d1f5b..898cc0e2 100644 --- a/public/stylesheets/style.css +++ b/public/stylesheets/style.css @@ -11,6 +11,7 @@ a { height: 100vh; width: fit-content; text-align: center; + text-align: -moz-center; justify-content: center; margin: 0 auto; padding-top: 30vh; diff --git a/service/controller-service.ts b/service/controller-service.ts index f1eabbeb..de7cf9dd 100644 --- a/service/controller-service.ts +++ b/service/controller-service.ts @@ -1,7 +1,8 @@ import Koa from 'koa' import Blacklist from './controllers/blacklist' -import config from './config-parser' +import config from './util/config-parser' import Ratelimiter from './controllers/rate-limiter' +import Waf from './controllers/waf' export const controller: Koa.Middleware = async function ( ctx: Koa.ParameterizedContext, @@ -9,24 +10,35 @@ export const controller: Koa.Middleware = async function ( ) { const blacklist = Blacklist.getInstance() const rateLimiter = Ratelimiter.getInstance() - if ((await blacklist.check(ctx.ip)) || !config.rate_limit) { - if (!!ctx.session.authorized || !config.pow) { - if (config.rate_limit) { - await Object.assign( - ctx.session, - await rateLimiter.process(ctx.ip, ctx.session) - ) - } - next() - } else { - console.log(ctx.request.url) - if (ctx.request.url == '/') { - ctx.redirect('/pow') - } else if (ctx.request.url == '/pow') { - await next() + const waf = Waf.getInstance() + if (await blacklist.check(ctx.ip)) { + const scanResult = await waf.scan(ctx) + if (!scanResult) { + if (!!ctx.session.authorized || !config.pow) { + if (config.rate_limit) { + await Object.assign( + ctx.session, + await rateLimiter.process(ctx.ip, ctx.session) + ) + } + next() } else { - ctx.redirect(`/pow?redirect=${ctx.request.url}`) + if (ctx.request.url == '/') { + ctx.redirect('/pow') + } else if (ctx.request.url == '/pow') { + await next() + } else { + ctx.redirect(`/pow?redirect=${ctx.request.url}`) + } } + } else { + console.log( + `Rule ${scanResult.id}: "${scanResult.cmt}" in category "${ + scanResult.type + }" has been triggered by request ${scanResult.location} at ${new Date().toISOString()}` + ) + ctx.status = 403 + await ctx.render('waf') } } else { ctx.status = 403 diff --git a/service/config-parser.ts b/service/util/config-parser.ts similarity index 80% rename from service/config-parser.ts rename to service/util/config-parser.ts index 2ccf8f7d..c62bcf10 100644 --- a/service/config-parser.ts +++ b/service/util/config-parser.ts @@ -4,7 +4,6 @@ dotenv.config() interface Config { session_key: string - waf: boolean pow: boolean nonce_validity: number initial_difficulty: number @@ -15,6 +14,10 @@ interface Config { rate_limit_ban_ip: boolean rate_limit_ip_threshold: number rate_limit_ban_minutes: number + waf: boolean + waf_url_exclude_rules: string + waf_header_exclude_rules: string + waf_body_exclude_rules: string } let config: Config @@ -22,7 +25,6 @@ let config: Config if (process.env.NODE_ENV === 'test') { config = { session_key: 'abcdefghijklmnop', - waf: true, pow: true, nonce_validity: 60000, initial_difficulty: 13, @@ -33,11 +35,14 @@ if (process.env.NODE_ENV === 'test') { rate_limit_ban_ip: true, rate_limit_ip_threshold: 3, rate_limit_ban_minutes: 0, + waf: true, + waf_url_exclude_rules: '', + waf_header_exclude_rules: '14,33,80,96,100', + waf_body_exclude_rules: '', } } else { config = { session_key: process.env.SESSION_KEY || 'abcdefghijklmnop', - waf: (process.env.WAF || 'on') == 'on', pow: (process.env.POW || 'on') == 'on', nonce_validity: parseInt(process.env.NONCE_VALIDITY || '') || 60 * 1000, initial_difficulty: parseInt(process.env.INITIAL_DIFFICULTY || '') || 13, @@ -56,6 +61,10 @@ if (process.env.NODE_ENV === 'test') { rate_limit_ban_minutes: parseInt( process.env.RATE_LIMIT_BAN_MINUTES || '15' ), + waf: (process.env.WAF || 'on') == 'on', + waf_url_exclude_rules: process.env.WAF_URL_EXCLUDE_RULES || '', + waf_header_exclude_rules: process.env.WAF_HEADER_EXCLUDE_RULES || '', + waf_body_exclude_rules: process.env.WAF_BODY_EXCLUDE_RULES || '', } } diff --git a/tests/integration/controller.test.ts b/tests/integration/controller.test.ts index ed93a835..5290fe7f 100644 --- a/tests/integration/controller.test.ts +++ b/tests/integration/controller.test.ts @@ -15,7 +15,7 @@ describe('Auth status', () => { await page.waitForFunction('window.nonceSent == true') await page.waitForNavigation() expect(await page.title()).toEqual('Example Domain') - }, 20000) + }, 10000) it('should be revocable', async () => { for (let i = 0; i < 3; i++) { @@ -35,11 +35,25 @@ describe('IP banning', () => { } expect(await page.content()).toContain('banned') await page.goto('http://localhost:3000/test?action=triggerRemoveExpired') - await page.goto('http://localhost:3000') + const response = await page.goto('http://localhost:3000') + expect(response.status()).toEqual(200) expect(await page.content()).not.toContain('banned') }, 20000) }) +describe('WAF', () => { + it('should be triggered on malicious request', async () => { + await page.waitForFunction('window.nonceSent == true') + await page.waitForNavigation() + expect(await page.title()).toEqual('Example Domain') + const response = await page.goto( + 'http://localhost:3000/select column from users' + ) + expect(response.status()).toEqual(403) + expect(await page.content()).toContain('WAF rules') + }, 10000) +}) + afterEach(async () => { await browser.close() }) diff --git a/tests/unit/config.spec.ts b/tests/unit/config.spec.ts index dfbc42b2..12a8ffae 100644 --- a/tests/unit/config.spec.ts +++ b/tests/unit/config.spec.ts @@ -1,4 +1,4 @@ -import config from '../../service/config-parser' +import config from '../../service/util/config-parser' describe('Node env', () => { it('should be "test"', async () => { @@ -10,7 +10,6 @@ describe('Configuration', () => { it('should match test spec', async () => { expect(config).toMatchObject({ session_key: 'abcdefghijklmnop', - waf: true, pow: true, nonce_validity: 60000, initial_difficulty: 13, @@ -21,6 +20,10 @@ describe('Configuration', () => { rate_limit_ban_ip: true, rate_limit_ip_threshold: 3, rate_limit_ban_minutes: 0, + waf: true, + waf_url_exclude_rules: '', + waf_header_exclude_rules: '14,33,80,96,100', + waf_body_exclude_rules: '', }) }) }) diff --git a/tests/unit/waf.test.ts b/tests/unit/waf.test.ts new file mode 100644 index 00000000..81be8bd9 --- /dev/null +++ b/tests/unit/waf.test.ts @@ -0,0 +1,19 @@ +import Waf from '../../service/controllers/waf' + +let waf: Waf + +beforeAll(() => { + waf = Waf.getInstance() +}) + +describe(`WAF`, () => { + it('should detect malicious string', async () => { + expect(await waf.test('select column from database', [])).toEqual('116') + }) + it('should return first rule triggered', async () => { + expect(await waf.test('select column from database where column like %asdf%', [])).toEqual('37') + }) + it('should ignore malicious string if excluded', async () => { + expect(await waf.test('select column from database', [116])).toEqual(0) + }) +}) diff --git a/views/waf.pug b/views/waf.pug new file mode 100644 index 00000000..dfb3dc7b --- /dev/null +++ b/views/waf.pug @@ -0,0 +1,13 @@ +doctype html +html + head + meta(charset='utf-8') + meta(http-equiv='X-UA-Compatible' content='IE=edge') + meta(name='viewport' content='width=device-width, initial-scale=1') + title PoW Shield + link(rel='stylesheet', href='/stylesheets/style.css') + script(src='/javascripts/bundle.min.js') + body + .container + h1 PoW Shield + p Your request has triggered WAF rules and has been blocked From 09b8e8992671e47cd44777dbf29e00eebb6e87fe Mon Sep 17 00:00:00 2001 From: RuiSiang <49300088+RuiSiang@users.noreply.github.com> Date: Tue, 23 Mar 2021 11:38:08 +0800 Subject: [PATCH 3/3] remove node 10.x from tests --- .github/workflows/njsscan-analysis.yml | 1 - .github/workflows/nodejs-ci.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/njsscan-analysis.yml b/.github/workflows/njsscan-analysis.yml index 06c4a574..b4fcf0dd 100644 --- a/.github/workflows/njsscan-analysis.yml +++ b/.github/workflows/njsscan-analysis.yml @@ -15,7 +15,6 @@ on: jobs: njsscan: runs-on: ubuntu-latest - name: njsscan code scanning steps: - name: Checkout the code uses: actions/checkout@v2 diff --git a/.github/workflows/nodejs-ci.yml b/.github/workflows/nodejs-ci.yml index d61cf892..5a907cff 100644 --- a/.github/workflows/nodejs-ci.yml +++ b/.github/workflows/nodejs-ci.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - node-version: [10.x, 12.x, 14.x] + node-version: [12.x, 14.x] steps: - uses: actions/checkout@v2