mirror of
https://github.com/ZeppelinBot/Zeppelin.git
synced 2025-03-16 22:21:51 +00:00
feat: Phisherman integration
This commit is contained in:
parent
f92ee9ba4f
commit
13c94a81cc
18 changed files with 681 additions and 17 deletions
|
@ -4,3 +4,4 @@ DB_USER=
|
||||||
DB_PASSWORD=
|
DB_PASSWORD=
|
||||||
DB_DATABASE=
|
DB_DATABASE=
|
||||||
PROFILING=false
|
PROFILING=false
|
||||||
|
PHISHERMAN_API_KEY=
|
||||||
|
|
120
backend/package-lock.json
generated
120
backend/package-lock.json
generated
|
@ -36,6 +36,7 @@
|
||||||
"lodash.pick": "^4.4.0",
|
"lodash.pick": "^4.4.0",
|
||||||
"moment-timezone": "^0.5.21",
|
"moment-timezone": "^0.5.21",
|
||||||
"mysql": "^2.16.0",
|
"mysql": "^2.16.0",
|
||||||
|
"node-fetch": "^2.6.5",
|
||||||
"parse-color": "^1.0.0",
|
"parse-color": "^1.0.0",
|
||||||
"passport": "^0.4.0",
|
"passport": "^0.4.0",
|
||||||
"passport-custom": "^1.0.5",
|
"passport-custom": "^1.0.5",
|
||||||
|
@ -66,6 +67,7 @@
|
||||||
"@types/lodash.at": "^4.6.3",
|
"@types/lodash.at": "^4.6.3",
|
||||||
"@types/moment-timezone": "^0.5.6",
|
"@types/moment-timezone": "^0.5.6",
|
||||||
"@types/node": "^14.0.14",
|
"@types/node": "^14.0.14",
|
||||||
|
"@types/node-fetch": "^2.5.12",
|
||||||
"@types/passport": "^1.0.0",
|
"@types/passport": "^1.0.0",
|
||||||
"@types/passport-oauth2": "^1.4.8",
|
"@types/passport-oauth2": "^1.4.8",
|
||||||
"@types/passport-strategy": "^0.2.35",
|
"@types/passport-strategy": "^0.2.35",
|
||||||
|
@ -75,8 +77,7 @@
|
||||||
"ava": "^3.10.0",
|
"ava": "^3.10.0",
|
||||||
"rimraf": "^2.6.2",
|
"rimraf": "^2.6.2",
|
||||||
"source-map-support": "^0.5.16",
|
"source-map-support": "^0.5.16",
|
||||||
"tsc-watch": "^4.0.0",
|
"tsc-watch": "^4.0.0"
|
||||||
"typescript": "^4.3.4"
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"../../Knub": {
|
"../../Knub": {
|
||||||
|
@ -424,6 +425,16 @@
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
|
||||||
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
|
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
|
||||||
},
|
},
|
||||||
|
"node_modules/@types/node-fetch": {
|
||||||
|
"version": "2.5.12",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
|
||||||
|
"integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
|
||||||
|
"dev": true,
|
||||||
|
"dependencies": {
|
||||||
|
"@types/node": "*",
|
||||||
|
"form-data": "^3.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/@types/normalize-package-data": {
|
"node_modules/@types/normalize-package-data": {
|
||||||
"version": "2.4.0",
|
"version": "2.4.0",
|
||||||
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
|
||||||
|
@ -2436,6 +2447,20 @@
|
||||||
"node": ">=6"
|
"node": ">=6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/form-data": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
|
||||||
|
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
|
||||||
|
"dev": true,
|
||||||
|
"dependencies": {
|
||||||
|
"asynckit": "^0.4.0",
|
||||||
|
"combined-stream": "^1.0.8",
|
||||||
|
"mime-types": "^2.1.12"
|
||||||
|
},
|
||||||
|
"engines": {
|
||||||
|
"node": ">= 6"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/forwarded": {
|
"node_modules/forwarded": {
|
||||||
"version": "0.1.2",
|
"version": "0.1.2",
|
||||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
||||||
|
@ -3513,9 +3538,12 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"node_modules/node-fetch": {
|
"node_modules/node-fetch": {
|
||||||
"version": "2.6.1",
|
"version": "2.6.5",
|
||||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
|
||||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
|
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
|
||||||
|
"dependencies": {
|
||||||
|
"whatwg-url": "^5.0.0"
|
||||||
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": "4.x || >=6.0.0"
|
"node": "4.x || >=6.0.0"
|
||||||
}
|
}
|
||||||
|
@ -5127,6 +5155,11 @@
|
||||||
"node": ">=0.6"
|
"node": ">=0.6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/tr46": {
|
||||||
|
"version": "0.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||||
|
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
|
||||||
|
},
|
||||||
"node_modules/trim-off-newlines": {
|
"node_modules/trim-off-newlines": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
|
||||||
|
@ -5430,9 +5463,10 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"node_modules/typescript": {
|
"node_modules/typescript": {
|
||||||
"version": "4.3.4",
|
"version": "4.4.4",
|
||||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
|
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
|
||||||
"integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==",
|
"integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
|
||||||
|
"peer": true,
|
||||||
"bin": {
|
"bin": {
|
||||||
"tsc": "bin/tsc",
|
"tsc": "bin/tsc",
|
||||||
"tsserver": "bin/tsserver"
|
"tsserver": "bin/tsserver"
|
||||||
|
@ -5647,6 +5681,11 @@
|
||||||
"defaults": "^1.0.3"
|
"defaults": "^1.0.3"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/webidl-conversions": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||||
|
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
|
||||||
|
},
|
||||||
"node_modules/well-known-symbols": {
|
"node_modules/well-known-symbols": {
|
||||||
"version": "2.0.0",
|
"version": "2.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
|
||||||
|
@ -5656,6 +5695,15 @@
|
||||||
"node": ">=6"
|
"node": ">=6"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/whatwg-url": {
|
||||||
|
"version": "5.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||||
|
"integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
|
||||||
|
"dependencies": {
|
||||||
|
"tr46": "~0.0.3",
|
||||||
|
"webidl-conversions": "^3.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/which": {
|
"node_modules/which": {
|
||||||
"version": "1.3.1",
|
"version": "1.3.1",
|
||||||
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
||||||
|
@ -6344,6 +6392,16 @@
|
||||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
|
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
|
||||||
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
|
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
|
||||||
},
|
},
|
||||||
|
"@types/node-fetch": {
|
||||||
|
"version": "2.5.12",
|
||||||
|
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
|
||||||
|
"integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"@types/node": "*",
|
||||||
|
"form-data": "^3.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"@types/normalize-package-data": {
|
"@types/normalize-package-data": {
|
||||||
"version": "2.4.0",
|
"version": "2.4.0",
|
||||||
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
|
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
|
||||||
|
@ -7918,6 +7976,17 @@
|
||||||
"locate-path": "^3.0.0"
|
"locate-path": "^3.0.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"form-data": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
|
||||||
|
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
|
||||||
|
"dev": true,
|
||||||
|
"requires": {
|
||||||
|
"asynckit": "^0.4.0",
|
||||||
|
"combined-stream": "^1.0.8",
|
||||||
|
"mime-types": "^2.1.12"
|
||||||
|
}
|
||||||
|
},
|
||||||
"forwarded": {
|
"forwarded": {
|
||||||
"version": "0.1.2",
|
"version": "0.1.2",
|
||||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
||||||
|
@ -8747,9 +8816,12 @@
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
"node-fetch": {
|
"node-fetch": {
|
||||||
"version": "2.6.1",
|
"version": "2.6.5",
|
||||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
|
||||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
|
||||||
|
"requires": {
|
||||||
|
"whatwg-url": "^5.0.0"
|
||||||
|
}
|
||||||
},
|
},
|
||||||
"node-gyp-build": {
|
"node-gyp-build": {
|
||||||
"version": "4.2.3",
|
"version": "4.2.3",
|
||||||
|
@ -9992,6 +10064,11 @@
|
||||||
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
|
||||||
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
|
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
|
||||||
},
|
},
|
||||||
|
"tr46": {
|
||||||
|
"version": "0.0.3",
|
||||||
|
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||||
|
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
|
||||||
|
},
|
||||||
"trim-off-newlines": {
|
"trim-off-newlines": {
|
||||||
"version": "1.0.1",
|
"version": "1.0.1",
|
||||||
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
|
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
|
||||||
|
@ -10220,9 +10297,10 @@
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"typescript": {
|
"typescript": {
|
||||||
"version": "4.3.4",
|
"version": "4.4.4",
|
||||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
|
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
|
||||||
"integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew=="
|
"integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
|
||||||
|
"peer": true
|
||||||
},
|
},
|
||||||
"uid2": {
|
"uid2": {
|
||||||
"version": "0.0.3",
|
"version": "0.0.3",
|
||||||
|
@ -10382,12 +10460,26 @@
|
||||||
"defaults": "^1.0.3"
|
"defaults": "^1.0.3"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"webidl-conversions": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||||
|
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
|
||||||
|
},
|
||||||
"well-known-symbols": {
|
"well-known-symbols": {
|
||||||
"version": "2.0.0",
|
"version": "2.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
|
||||||
"integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==",
|
"integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==",
|
||||||
"dev": true
|
"dev": true
|
||||||
},
|
},
|
||||||
|
"whatwg-url": {
|
||||||
|
"version": "5.0.0",
|
||||||
|
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||||
|
"integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
|
||||||
|
"requires": {
|
||||||
|
"tr46": "~0.0.3",
|
||||||
|
"webidl-conversions": "^3.0.0"
|
||||||
|
}
|
||||||
|
},
|
||||||
"which": {
|
"which": {
|
||||||
"version": "1.3.1",
|
"version": "1.3.1",
|
||||||
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
||||||
|
|
|
@ -51,6 +51,7 @@
|
||||||
"lodash.pick": "^4.4.0",
|
"lodash.pick": "^4.4.0",
|
||||||
"moment-timezone": "^0.5.21",
|
"moment-timezone": "^0.5.21",
|
||||||
"mysql": "^2.16.0",
|
"mysql": "^2.16.0",
|
||||||
|
"node-fetch": "^2.6.5",
|
||||||
"parse-color": "^1.0.0",
|
"parse-color": "^1.0.0",
|
||||||
"passport": "^0.4.0",
|
"passport": "^0.4.0",
|
||||||
"passport-custom": "^1.0.5",
|
"passport-custom": "^1.0.5",
|
||||||
|
@ -81,6 +82,7 @@
|
||||||
"@types/lodash.at": "^4.6.3",
|
"@types/lodash.at": "^4.6.3",
|
||||||
"@types/moment-timezone": "^0.5.6",
|
"@types/moment-timezone": "^0.5.6",
|
||||||
"@types/node": "^14.0.14",
|
"@types/node": "^14.0.14",
|
||||||
|
"@types/node-fetch": "^2.5.12",
|
||||||
"@types/passport": "^1.0.0",
|
"@types/passport": "^1.0.0",
|
||||||
"@types/passport-oauth2": "^1.4.8",
|
"@types/passport-oauth2": "^1.4.8",
|
||||||
"@types/passport-strategy": "^0.2.35",
|
"@types/passport-strategy": "^0.2.35",
|
||||||
|
@ -90,8 +92,7 @@
|
||||||
"ava": "^3.10.0",
|
"ava": "^3.10.0",
|
||||||
"rimraf": "^2.6.2",
|
"rimraf": "^2.6.2",
|
||||||
"source-map-support": "^0.5.16",
|
"source-map-support": "^0.5.16",
|
||||||
"tsc-watch": "^4.0.0",
|
"tsc-watch": "^4.0.0"
|
||||||
"typescript": "^4.3.4"
|
|
||||||
},
|
},
|
||||||
"ava": {
|
"ava": {
|
||||||
"files": [
|
"files": [
|
||||||
|
|
229
backend/src/data/Phisherman.ts
Normal file
229
backend/src/data/Phisherman.ts
Normal file
|
@ -0,0 +1,229 @@
|
||||||
|
import { getRepository, Repository } from "typeorm";
|
||||||
|
import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry";
|
||||||
|
import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman";
|
||||||
|
import fetch, { Headers } from "node-fetch";
|
||||||
|
import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
|
||||||
|
import moment from "moment-timezone";
|
||||||
|
import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
|
||||||
|
import crypto from "crypto";
|
||||||
|
|
||||||
|
const API_URL = "https://api.phisherman.gg";
|
||||||
|
const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
|
||||||
|
|
||||||
|
let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
|
||||||
|
|
||||||
|
const pendingApiRequests: Map<string, Promise<unknown>> = new Map();
|
||||||
|
const pendingDomainInfoChecks: Map<string, Promise<PhishermanDomainInfo | null>> = new Map();
|
||||||
|
|
||||||
|
type MemoryCacheEntry = {
|
||||||
|
info: PhishermanDomainInfo | null;
|
||||||
|
expires: number;
|
||||||
|
};
|
||||||
|
const memoryCache: Map<string, MemoryCacheEntry> = new Map();
|
||||||
|
|
||||||
|
setInterval(() => {
|
||||||
|
const now = Date.now();
|
||||||
|
for (const [key, entry] of memoryCache.entries()) {
|
||||||
|
if (entry.expires <= now) {
|
||||||
|
memoryCache.delete(key);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}, 2 * MINUTES);
|
||||||
|
|
||||||
|
const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES;
|
||||||
|
const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES;
|
||||||
|
const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS;
|
||||||
|
|
||||||
|
const KEY_VALIDITY_LIFETIME = 24 * HOURS;
|
||||||
|
|
||||||
|
let cacheRepository: Repository<PhishermanCacheEntry> | null = null;
|
||||||
|
function getCacheRepository(): Repository<PhishermanCacheEntry> {
|
||||||
|
if (cacheRepository == null) {
|
||||||
|
cacheRepository = getRepository(PhishermanCacheEntry);
|
||||||
|
}
|
||||||
|
return cacheRepository;
|
||||||
|
}
|
||||||
|
|
||||||
|
let keyCacheRepository: Repository<PhishermanKeyCacheEntry> | null = null;
|
||||||
|
function getKeyCacheRepository(): Repository<PhishermanKeyCacheEntry> {
|
||||||
|
if (keyCacheRepository == null) {
|
||||||
|
keyCacheRepository = getRepository(PhishermanKeyCacheEntry);
|
||||||
|
}
|
||||||
|
return keyCacheRepository;
|
||||||
|
}
|
||||||
|
|
||||||
|
class PhishermanApiError extends Error {
|
||||||
|
status: number;
|
||||||
|
constructor(status: number, message: string) {
|
||||||
|
super(message);
|
||||||
|
this.status = status;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function hasPhishermanMasterAPIKey() {
|
||||||
|
return MASTER_API_KEY != null && MASTER_API_KEY !== "";
|
||||||
|
}
|
||||||
|
|
||||||
|
export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean {
|
||||||
|
return info.classification === "safe";
|
||||||
|
}
|
||||||
|
|
||||||
|
const leadingSlashRegex = /^\/+/g;
|
||||||
|
function trimLeadingSlash(str: string): string {
|
||||||
|
return str.replace(leadingSlashRegex, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Make an arbitrary API call to the Phisherman API
|
||||||
|
*/
|
||||||
|
async function apiCall<T>(
|
||||||
|
method: "GET" | "POST",
|
||||||
|
resource: string,
|
||||||
|
payload?: Record<string, unknown> | null,
|
||||||
|
): Promise<T> {
|
||||||
|
if (!hasPhishermanMasterAPIKey()) {
|
||||||
|
throw new Error("Phisherman master API key missing");
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = `${API_URL}/${trimLeadingSlash(resource)}`;
|
||||||
|
const key = `${method} ${url}`;
|
||||||
|
|
||||||
|
if (pendingApiRequests.has(key)) {
|
||||||
|
return pendingApiRequests.get(key)! as Promise<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const requestPromise = (async () => {
|
||||||
|
const response = await fetch(url, {
|
||||||
|
method,
|
||||||
|
headers: new Headers({
|
||||||
|
Accept: "application/json",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
Authorization: `Bearer ${MASTER_API_KEY}`,
|
||||||
|
}),
|
||||||
|
body: payload ? JSON.stringify(payload) : undefined,
|
||||||
|
});
|
||||||
|
const data = await response.json().catch(() => null);
|
||||||
|
if (!response.ok || (data as any)?.success === false) {
|
||||||
|
throw new PhishermanApiError(response.status, (data as any)?.message ?? "");
|
||||||
|
}
|
||||||
|
return data;
|
||||||
|
})();
|
||||||
|
requestPromise.finally(() => {
|
||||||
|
pendingApiRequests.delete(key);
|
||||||
|
});
|
||||||
|
pendingApiRequests.set(key, requestPromise);
|
||||||
|
return requestPromise as Promise<T>;
|
||||||
|
}
|
||||||
|
|
||||||
|
type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo;
|
||||||
|
async function fetchDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
||||||
|
// tslint:disable-next-line:no-console
|
||||||
|
console.log(`[PHISHERMAN] Requesting domain information: ${domain}`);
|
||||||
|
const result = await apiCall<Record<string, DomainInfoApiCallResult>>("GET", `/v2/domains/info/${domain}`);
|
||||||
|
const domainInfo = result[domain];
|
||||||
|
if (domainInfo.classification === "unknown") {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return domainInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getPhishermanDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
||||||
|
if (pendingDomainInfoChecks.has(domain)) {
|
||||||
|
return pendingDomainInfoChecks.get(domain)!;
|
||||||
|
}
|
||||||
|
|
||||||
|
const promise = (async () => {
|
||||||
|
if (memoryCache.has(domain)) {
|
||||||
|
return memoryCache.get(domain)!.info;
|
||||||
|
}
|
||||||
|
|
||||||
|
const dbCache = getCacheRepository();
|
||||||
|
const existingCachedEntry = await dbCache.findOne({ domain });
|
||||||
|
if (existingCachedEntry) {
|
||||||
|
return existingCachedEntry.data;
|
||||||
|
}
|
||||||
|
|
||||||
|
const freshData = await fetchDomainInfo(domain);
|
||||||
|
const expiryTime =
|
||||||
|
freshData === null
|
||||||
|
? UNKNOWN_DOMAIN_CACHE_LIFETIME
|
||||||
|
: phishermanDomainIsSafe(freshData)
|
||||||
|
? SAFE_DOMAIN_CACHE_LIFETIME
|
||||||
|
: DETECTED_DOMAIN_CACHE_LIFETIME;
|
||||||
|
memoryCache.set(domain, {
|
||||||
|
info: freshData,
|
||||||
|
expires: Date.now() + expiryTime,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (freshData) {
|
||||||
|
// Database cache only stores safe/detected domains, not unknown ones
|
||||||
|
await dbCache.insert({
|
||||||
|
domain,
|
||||||
|
data: freshData,
|
||||||
|
expires_at: moment().add(expiryTime, "ms").format(DBDateFormat),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
return freshData;
|
||||||
|
})();
|
||||||
|
promise.finally(() => {
|
||||||
|
pendingDomainInfoChecks.delete(domain);
|
||||||
|
});
|
||||||
|
pendingDomainInfoChecks.set(domain, promise);
|
||||||
|
|
||||||
|
return promise;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function phishermanApiKeyIsValid(apiKey: string): Promise<boolean> {
|
||||||
|
const keyCache = getKeyCacheRepository();
|
||||||
|
const hash = crypto.createHash("sha256").update(apiKey).digest("hex");
|
||||||
|
const entry = await keyCache.findOne({ hash });
|
||||||
|
if (entry) {
|
||||||
|
return entry.is_valid;
|
||||||
|
}
|
||||||
|
|
||||||
|
const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey });
|
||||||
|
|
||||||
|
await keyCache.insert({
|
||||||
|
hash,
|
||||||
|
is_valid: isValid,
|
||||||
|
expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat),
|
||||||
|
});
|
||||||
|
|
||||||
|
return isValid;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function trackPhishermanCaughtDomain(apiKey: string, domain: string) {
|
||||||
|
if (!caughtDomainTrackingMap.has(apiKey)) {
|
||||||
|
caughtDomainTrackingMap.set(apiKey, new Map());
|
||||||
|
}
|
||||||
|
const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!;
|
||||||
|
if (!apiKeyMap.has(domain)) {
|
||||||
|
apiKeyMap.set(domain, []);
|
||||||
|
}
|
||||||
|
const timestamps = apiKeyMap.get(domain)!;
|
||||||
|
timestamps.push(Date.now());
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function reportTrackedDomainsToPhisherman() {
|
||||||
|
const result = {};
|
||||||
|
for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) {
|
||||||
|
result[apiKey] = {};
|
||||||
|
for (const [domain, timestamps] of domains.entries()) {
|
||||||
|
result[apiKey][domain] = timestamps;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Object.keys(result).length > 0) {
|
||||||
|
await apiCall("POST", "/v2/phish/caught/bulk", result);
|
||||||
|
caughtDomainTrackingMap = new Map();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteStalePhishermanCacheEntries() {
|
||||||
|
await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function deleteStalePhishermanKeyCacheEntries() {
|
||||||
|
await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
||||||
|
}
|
18
backend/src/data/entities/PhishermanCacheEntry.ts
Normal file
18
backend/src/data/entities/PhishermanCacheEntry.ts
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
import { Column, Entity, PrimaryColumn } from "typeorm";
|
||||||
|
import { PhishermanDomainInfo } from "../types/phisherman";
|
||||||
|
|
||||||
|
@Entity("phisherman_cache")
|
||||||
|
export class PhishermanCacheEntry {
|
||||||
|
@Column()
|
||||||
|
@PrimaryColumn()
|
||||||
|
id: number;
|
||||||
|
|
||||||
|
@Column()
|
||||||
|
domain: string;
|
||||||
|
|
||||||
|
@Column("simple-json")
|
||||||
|
data: PhishermanDomainInfo;
|
||||||
|
|
||||||
|
@Column()
|
||||||
|
expires_at: string;
|
||||||
|
}
|
17
backend/src/data/entities/PhishermanKeyCacheEntry.ts
Normal file
17
backend/src/data/entities/PhishermanKeyCacheEntry.ts
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
import { Column, Entity, PrimaryColumn } from "typeorm";
|
||||||
|
|
||||||
|
@Entity("phisherman_key_cache")
|
||||||
|
export class PhishermanKeyCacheEntry {
|
||||||
|
@Column()
|
||||||
|
@PrimaryColumn()
|
||||||
|
id: number;
|
||||||
|
|
||||||
|
@Column()
|
||||||
|
hash: string;
|
||||||
|
|
||||||
|
@Column()
|
||||||
|
is_valid: boolean;
|
||||||
|
|
||||||
|
@Column()
|
||||||
|
expires_at: string;
|
||||||
|
}
|
26
backend/src/data/loops/phishermanLoops.ts
Normal file
26
backend/src/data/loops/phishermanLoops.ts
Normal file
|
@ -0,0 +1,26 @@
|
||||||
|
import { HOURS, MINUTES } from "../../utils";
|
||||||
|
import {
|
||||||
|
deleteStalePhishermanCacheEntries,
|
||||||
|
deleteStalePhishermanKeyCacheEntries,
|
||||||
|
reportTrackedDomainsToPhisherman,
|
||||||
|
} from "../Phisherman";
|
||||||
|
|
||||||
|
const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES;
|
||||||
|
const REPORT_LOOP_INTERVAL = 15 * MINUTES;
|
||||||
|
|
||||||
|
export async function runPhishermanCacheCleanupLoop() {
|
||||||
|
console.log("[PHISHERMAN] Deleting stale cache entries");
|
||||||
|
await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err));
|
||||||
|
|
||||||
|
console.log("[PHISHERMAN] Deleting stale key cache entries");
|
||||||
|
await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err));
|
||||||
|
|
||||||
|
setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function runPhishermanReportingLoop() {
|
||||||
|
console.log("[PHISHERMAN] Reporting tracked domains");
|
||||||
|
await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err));
|
||||||
|
|
||||||
|
setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL);
|
||||||
|
}
|
32
backend/src/data/types/phisherman.ts
Normal file
32
backend/src/data/types/phisherman.ts
Normal file
|
@ -0,0 +1,32 @@
|
||||||
|
export interface PhishermanUnknownDomain {
|
||||||
|
classification: "unknown";
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PhishermanDomainInfo {
|
||||||
|
status: string;
|
||||||
|
lastChecked: string;
|
||||||
|
verifiedPhish: boolean;
|
||||||
|
classification: "safe" | "malicious";
|
||||||
|
created: string;
|
||||||
|
firstSeen: string | null;
|
||||||
|
lastSeen: string | null;
|
||||||
|
targetedBrand: string;
|
||||||
|
phishCaught: number;
|
||||||
|
details: PhishermanDomainInfoDetails;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PhishermanDomainInfoDetails {
|
||||||
|
phishTankId: string | null;
|
||||||
|
urlScanId: string;
|
||||||
|
websiteScreenshot: string;
|
||||||
|
ip_address: string;
|
||||||
|
asn: PhishermanDomainInfoAsn;
|
||||||
|
registry: string;
|
||||||
|
country: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PhishermanDomainInfoAsn {
|
||||||
|
asn: string;
|
||||||
|
asn_name: string;
|
||||||
|
route: string;
|
||||||
|
}
|
|
@ -33,6 +33,8 @@ import { runSavedMessageCleanupLoop } from "./data/loops/savedMessageCleanupLoop
|
||||||
import { performance } from "perf_hooks";
|
import { performance } from "perf_hooks";
|
||||||
import { setProfiler } from "./profiler";
|
import { setProfiler } from "./profiler";
|
||||||
import { enableProfiling } from "./utils/easyProfiler";
|
import { enableProfiling } from "./utils/easyProfiler";
|
||||||
|
import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./data/loops/phishermanLoops";
|
||||||
|
import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
|
||||||
|
|
||||||
if (!process.env.KEY) {
|
if (!process.env.KEY) {
|
||||||
// tslint:disable-next-line:no-console
|
// tslint:disable-next-line:no-console
|
||||||
|
@ -363,6 +365,13 @@ connect().then(async () => {
|
||||||
runExpiredArchiveDeletionLoop();
|
runExpiredArchiveDeletionLoop();
|
||||||
await sleep(10 * SECONDS);
|
await sleep(10 * SECONDS);
|
||||||
runSavedMessageCleanupLoop();
|
runSavedMessageCleanupLoop();
|
||||||
|
|
||||||
|
if (hasPhishermanMasterAPIKey()) {
|
||||||
|
await sleep(10 * SECONDS);
|
||||||
|
runPhishermanCacheCleanupLoop();
|
||||||
|
await sleep(10 * SECONDS);
|
||||||
|
runPhishermanReportingLoop();
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
setProfiler(bot.profiler);
|
setProfiler(bot.profiler);
|
||||||
|
|
|
@ -0,0 +1,38 @@
|
||||||
|
import { MigrationInterface, QueryRunner, Table } from "typeorm";
|
||||||
|
|
||||||
|
export class CreatePhishermanCacheTable1634563901575 implements MigrationInterface {
|
||||||
|
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.createTable(
|
||||||
|
new Table({
|
||||||
|
name: "phisherman_cache",
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
name: "id",
|
||||||
|
type: "int",
|
||||||
|
isPrimary: true,
|
||||||
|
isGenerated: true,
|
||||||
|
generationStrategy: "increment",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "domain",
|
||||||
|
type: "varchar",
|
||||||
|
length: "255",
|
||||||
|
isUnique: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "data",
|
||||||
|
type: "text",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "expires_at",
|
||||||
|
type: "datetime",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.dropTable("phisherman_cache");
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,38 @@
|
||||||
|
import { MigrationInterface, QueryRunner, Table } from "typeorm";
|
||||||
|
|
||||||
|
export class CreatePhishermanKeyCacheTable1635596150234 implements MigrationInterface {
|
||||||
|
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.createTable(
|
||||||
|
new Table({
|
||||||
|
name: "phisherman_key_cache",
|
||||||
|
columns: [
|
||||||
|
{
|
||||||
|
name: "id",
|
||||||
|
type: "int",
|
||||||
|
isPrimary: true,
|
||||||
|
isGenerated: true,
|
||||||
|
generationStrategy: "increment",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "hash",
|
||||||
|
type: "varchar",
|
||||||
|
length: "255",
|
||||||
|
isUnique: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "is_valid",
|
||||||
|
type: "tinyint",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "expires_at",
|
||||||
|
type: "datetime",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.dropTable("phisherman_key_cache");
|
||||||
|
}
|
||||||
|
}
|
|
@ -30,6 +30,7 @@ import { clearOldRecentSpam } from "./functions/clearOldRecentSpam";
|
||||||
import { pluginInfo } from "./info";
|
import { pluginInfo } from "./info";
|
||||||
import { availableTriggers } from "./triggers/availableTriggers";
|
import { availableTriggers } from "./triggers/availableTriggers";
|
||||||
import { AutomodPluginType, ConfigSchema } from "./types";
|
import { AutomodPluginType, ConfigSchema } from "./types";
|
||||||
|
import { PhishermanPlugin } from "../Phisherman/PhishermanPlugin";
|
||||||
|
|
||||||
const defaultOptions = {
|
const defaultOptions = {
|
||||||
config: {
|
config: {
|
||||||
|
@ -183,6 +184,7 @@ export const AutomodPlugin = zeppelinGuildPlugin<AutomodPluginType>()({
|
||||||
ModActionsPlugin,
|
ModActionsPlugin,
|
||||||
MutesPlugin,
|
MutesPlugin,
|
||||||
CountersPlugin,
|
CountersPlugin,
|
||||||
|
PhishermanPlugin,
|
||||||
],
|
],
|
||||||
|
|
||||||
configSchema: ConfigSchema,
|
configSchema: ConfigSchema,
|
||||||
|
|
|
@ -9,10 +9,13 @@ import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions
|
||||||
import { automodTrigger } from "../helpers";
|
import { automodTrigger } from "../helpers";
|
||||||
import { mergeRegexes } from "../../../utils/mergeRegexes";
|
import { mergeRegexes } from "../../../utils/mergeRegexes";
|
||||||
import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex";
|
import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex";
|
||||||
|
import { PhishermanPlugin } from "../../Phisherman/PhishermanPlugin";
|
||||||
|
import { phishermanDomainIsSafe } from "../../../data/Phisherman";
|
||||||
|
|
||||||
interface MatchResultType {
|
interface MatchResultType {
|
||||||
type: MatchableTextType;
|
type: MatchableTextType;
|
||||||
link: string;
|
link: string;
|
||||||
|
details?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const regexCache = new WeakMap<any, RegExp[]>();
|
const regexCache = new WeakMap<any, RegExp[]>();
|
||||||
|
@ -28,6 +31,12 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
|
||||||
exclude_words: tNullable(t.array(t.string)),
|
exclude_words: tNullable(t.array(t.string)),
|
||||||
include_regex: tNullable(t.array(TRegex)),
|
include_regex: tNullable(t.array(TRegex)),
|
||||||
exclude_regex: tNullable(t.array(TRegex)),
|
exclude_regex: tNullable(t.array(TRegex)),
|
||||||
|
phisherman: tNullable(
|
||||||
|
t.type({
|
||||||
|
include_suspected: tNullable(t.boolean),
|
||||||
|
include_verified: tNullable(t.boolean),
|
||||||
|
}),
|
||||||
|
),
|
||||||
only_real_links: t.boolean,
|
only_real_links: t.boolean,
|
||||||
match_messages: t.boolean,
|
match_messages: t.boolean,
|
||||||
match_embeds: t.boolean,
|
match_embeds: t.boolean,
|
||||||
|
@ -150,6 +159,25 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (trigger.phisherman) {
|
||||||
|
const phishermanResult = await pluginData.getPlugin(PhishermanPlugin).getDomainInfo(normalizedHostname);
|
||||||
|
if (phishermanResult != null && !phishermanDomainIsSafe(phishermanResult)) {
|
||||||
|
if (
|
||||||
|
(trigger.phisherman.include_suspected && !phishermanResult.verifiedPhish) ||
|
||||||
|
(trigger.phisherman.include_verified && phishermanResult.verifiedPhish)
|
||||||
|
) {
|
||||||
|
const suspectedVerified = phishermanResult.verifiedPhish ? "verified" : "suspected";
|
||||||
|
return {
|
||||||
|
extra: {
|
||||||
|
type,
|
||||||
|
link: link.input,
|
||||||
|
details: `using Phisherman (${suspectedVerified})`,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -158,6 +186,11 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
|
||||||
|
|
||||||
renderMatchInformation({ pluginData, contexts, matchResult }) {
|
renderMatchInformation({ pluginData, contexts, matchResult }) {
|
||||||
const partialSummary = getTextMatchPartialSummary(pluginData, matchResult.extra.type, contexts[0]);
|
const partialSummary = getTextMatchPartialSummary(pluginData, matchResult.extra.type, contexts[0]);
|
||||||
return `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\` in ${partialSummary}`;
|
let information = `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\``;
|
||||||
|
if (matchResult.extra.details) {
|
||||||
|
information += ` ${matchResult.extra.details}`;
|
||||||
|
}
|
||||||
|
information += ` in ${partialSummary}`;
|
||||||
|
return information;
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
49
backend/src/plugins/Phisherman/PhishermanPlugin.ts
Normal file
49
backend/src/plugins/Phisherman/PhishermanPlugin.ts
Normal file
|
@ -0,0 +1,49 @@
|
||||||
|
import { PluginOptions, typedGuildCommand } from "knub";
|
||||||
|
import { GuildPingableRoles } from "../../data/GuildPingableRoles";
|
||||||
|
import { zeppelinGuildPlugin } from "../ZeppelinPluginBlueprint";
|
||||||
|
import { ConfigSchema, PhishermanPluginType } from "./types";
|
||||||
|
import {
|
||||||
|
getPhishermanDomainInfo,
|
||||||
|
hasPhishermanMasterAPIKey,
|
||||||
|
phishermanApiKeyIsValid,
|
||||||
|
reportTrackedDomainsToPhisherman,
|
||||||
|
} from "../../data/Phisherman";
|
||||||
|
import { mapToPublicFn } from "../../pluginUtils";
|
||||||
|
import { getDomainInfo } from "./functions/getDomainInfo";
|
||||||
|
import { pluginInfo } from "./info";
|
||||||
|
|
||||||
|
const defaultOptions: PluginOptions<PhishermanPluginType> = {
|
||||||
|
config: {
|
||||||
|
api_key: null,
|
||||||
|
},
|
||||||
|
overrides: [],
|
||||||
|
};
|
||||||
|
|
||||||
|
export const PhishermanPlugin = zeppelinGuildPlugin<PhishermanPluginType>()({
|
||||||
|
name: "phisherman",
|
||||||
|
showInDocs: true,
|
||||||
|
info: pluginInfo,
|
||||||
|
|
||||||
|
configSchema: ConfigSchema,
|
||||||
|
defaultOptions,
|
||||||
|
|
||||||
|
// prettier-ignore
|
||||||
|
public: {
|
||||||
|
getDomainInfo: mapToPublicFn(getDomainInfo),
|
||||||
|
},
|
||||||
|
|
||||||
|
async beforeLoad(pluginData) {
|
||||||
|
pluginData.state.validApiKey = null;
|
||||||
|
|
||||||
|
if (!hasPhishermanMasterAPIKey()) {
|
||||||
|
// tslint:disable-next-line:no-console
|
||||||
|
console.warn("Could not load Phisherman plugin: master API key is missing");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const apiKey = pluginData.config.get().api_key;
|
||||||
|
if (apiKey && (await phishermanApiKeyIsValid(apiKey).catch(() => false))) {
|
||||||
|
pluginData.state.validApiKey = apiKey;
|
||||||
|
}
|
||||||
|
},
|
||||||
|
});
|
20
backend/src/plugins/Phisherman/functions/getDomainInfo.ts
Normal file
20
backend/src/plugins/Phisherman/functions/getDomainInfo.ts
Normal file
|
@ -0,0 +1,20 @@
|
||||||
|
import { GuildPluginData } from "knub";
|
||||||
|
import { PhishermanPluginType } from "../types";
|
||||||
|
import { PhishermanDomainInfo } from "../../../data/types/phisherman";
|
||||||
|
import { getPhishermanDomainInfo, phishermanDomainIsSafe, trackPhishermanCaughtDomain } from "../../../data/Phisherman";
|
||||||
|
|
||||||
|
export async function getDomainInfo(
|
||||||
|
pluginData: GuildPluginData<PhishermanPluginType>,
|
||||||
|
domain: string,
|
||||||
|
): Promise<PhishermanDomainInfo | null> {
|
||||||
|
if (!pluginData.state.validApiKey) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const info = await getPhishermanDomainInfo(domain).catch(() => null);
|
||||||
|
if (info != null && !phishermanDomainIsSafe(info)) {
|
||||||
|
trackPhishermanCaughtDomain(pluginData.state.validApiKey, domain);
|
||||||
|
}
|
||||||
|
|
||||||
|
return info;
|
||||||
|
}
|
41
backend/src/plugins/Phisherman/info.ts
Normal file
41
backend/src/plugins/Phisherman/info.ts
Normal file
|
@ -0,0 +1,41 @@
|
||||||
|
import { trimPluginDescription } from "../../utils";
|
||||||
|
import { ZeppelinGuildPluginBlueprint } from "../ZeppelinPluginBlueprint";
|
||||||
|
|
||||||
|
export const pluginInfo: ZeppelinGuildPluginBlueprint["info"] = {
|
||||||
|
prettyName: "Phisherman",
|
||||||
|
description: trimPluginDescription(`
|
||||||
|
Match scam/phishing links using the Phisherman API. See https://phisherman.gg/ for more details!
|
||||||
|
`),
|
||||||
|
configurationGuide: trimPluginDescription(`
|
||||||
|
### Getting started
|
||||||
|
To get started, request an API key for Phisherman following the instructions at https://docs.phisherman.gg/#/api/getting-started?id=requesting-api-access.
|
||||||
|
Then, add the api key to the plugin's config:
|
||||||
|
|
||||||
|
~~~yml
|
||||||
|
phisherman:
|
||||||
|
config:
|
||||||
|
api_key: "your key here"
|
||||||
|
~~~
|
||||||
|
|
||||||
|
### Note
|
||||||
|
When using Phisherman features in Zeppelin, Zeppelin reports statistics about checked links back to Phisherman. This only includes the domain (e.g. zeppelin.gg), not the full link.
|
||||||
|
|
||||||
|
### Usage with Automod
|
||||||
|
Once you have configured the Phisherman plugin, you are ready to use it with automod. Currently, Phisherman is available as an option in the \`match_links\` plugin:
|
||||||
|
|
||||||
|
~~~yml
|
||||||
|
automod:
|
||||||
|
config:
|
||||||
|
rules:
|
||||||
|
# Clean any scam links detected by Phisherman
|
||||||
|
filter_scam_links:
|
||||||
|
triggers:
|
||||||
|
- match_links:
|
||||||
|
phisherman:
|
||||||
|
include_suspected: true # It's recommended to keep this enabled to catch new scam domains quickly
|
||||||
|
include_verified: true
|
||||||
|
actions:
|
||||||
|
clean: true
|
||||||
|
~~~
|
||||||
|
`),
|
||||||
|
};
|
16
backend/src/plugins/Phisherman/types.ts
Normal file
16
backend/src/plugins/Phisherman/types.ts
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
import * as t from "io-ts";
|
||||||
|
import { BasePluginType } from "knub";
|
||||||
|
import { tNullable } from "../../utils";
|
||||||
|
|
||||||
|
export const ConfigSchema = t.type({
|
||||||
|
api_key: tNullable(t.string),
|
||||||
|
});
|
||||||
|
export type TConfigSchema = t.TypeOf<typeof ConfigSchema>;
|
||||||
|
|
||||||
|
export interface PhishermanPluginType extends BasePluginType {
|
||||||
|
config: TConfigSchema;
|
||||||
|
|
||||||
|
state: {
|
||||||
|
validApiKey: string | null;
|
||||||
|
};
|
||||||
|
}
|
|
@ -34,6 +34,7 @@ import { UsernameSaverPlugin } from "./UsernameSaver/UsernameSaverPlugin";
|
||||||
import { UtilityPlugin } from "./Utility/UtilityPlugin";
|
import { UtilityPlugin } from "./Utility/UtilityPlugin";
|
||||||
import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin";
|
import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin";
|
||||||
import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint";
|
import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint";
|
||||||
|
import { PhishermanPlugin } from "./Phisherman/PhishermanPlugin";
|
||||||
|
|
||||||
// prettier-ignore
|
// prettier-ignore
|
||||||
export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
|
export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
|
||||||
|
@ -69,6 +70,7 @@ export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
|
||||||
TimeAndDatePlugin,
|
TimeAndDatePlugin,
|
||||||
CountersPlugin,
|
CountersPlugin,
|
||||||
ContextMenuPlugin,
|
ContextMenuPlugin,
|
||||||
|
PhishermanPlugin,
|
||||||
];
|
];
|
||||||
|
|
||||||
// prettier-ignore
|
// prettier-ignore
|
||||||
|
|
Loading…
Add table
Reference in a new issue