mirror of
https://github.com/ZeppelinBot/Zeppelin.git
synced 2025-03-15 05:41:51 +00:00
feat: Phisherman integration
This commit is contained in:
Dragory
parent
f92ee9ba4f
commit
13c94a81cc
18 changed files with 681 additions and 17 deletions
|
|
@ -4,3 +4,4 @@ DB_USER=
|
|||
DB_PASSWORD=
|
||||
DB_DATABASE=
|
||||
PROFILING=false
|
||||
PHISHERMAN_API_KEY=
|
||||
|
|
|
|||
120
backend/package-lock.json
generated
120
backend/package-lock.json
generated
|
|
@ -36,6 +36,7 @@
|
|||
"lodash.pick": "^4.4.0",
|
||||
"moment-timezone": "^0.5.21",
|
||||
"mysql": "^2.16.0",
|
||||
"node-fetch": "^2.6.5",
|
||||
"parse-color": "^1.0.0",
|
||||
"passport": "^0.4.0",
|
||||
"passport-custom": "^1.0.5",
|
||||
|
|
@ -66,6 +67,7 @@
|
|||
"@types/lodash.at": "^4.6.3",
|
||||
"@types/moment-timezone": "^0.5.6",
|
||||
"@types/node": "^14.0.14",
|
||||
"@types/node-fetch": "^2.5.12",
|
||||
"@types/passport": "^1.0.0",
|
||||
"@types/passport-oauth2": "^1.4.8",
|
||||
"@types/passport-strategy": "^0.2.35",
|
||||
|
|
@ -75,8 +77,7 @@
|
|||
"ava": "^3.10.0",
|
||||
"rimraf": "^2.6.2",
|
||||
"source-map-support": "^0.5.16",
|
||||
"tsc-watch": "^4.0.0",
|
||||
"typescript": "^4.3.4"
|
||||
"tsc-watch": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"../../Knub": {
|
||||
|
|
@ -424,6 +425,16 @@
|
|||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
|
||||
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
|
||||
},
|
||||
"node_modules/@types/node-fetch": {
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
|
||||
"integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/normalize-package-data": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
|
||||
|
|
@ -2436,6 +2447,20 @@
|
|||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/form-data": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
|
||||
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/forwarded": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
||||
|
|
@ -3513,9 +3538,12 @@
|
|||
"dev": true
|
||||
},
|
||||
"node_modules/node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
|
||||
"version": "2.6.5",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
|
||||
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
|
||||
"dependencies": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": "4.x || >=6.0.0"
|
||||
}
|
||||
|
|
@ -5127,6 +5155,11 @@
|
|||
"node": ">=0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
|
||||
},
|
||||
"node_modules/trim-off-newlines": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
|
||||
|
|
@ -5430,9 +5463,10 @@
|
|||
}
|
||||
},
|
||||
"node_modules/typescript": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
|
||||
"integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==",
|
||||
"version": "4.4.4",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
|
||||
"integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
|
||||
"peer": true,
|
||||
"bin": {
|
||||
"tsc": "bin/tsc",
|
||||
"tsserver": "bin/tsserver"
|
||||
|
|
@ -5647,6 +5681,11 @@
|
|||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
|
||||
},
|
||||
"node_modules/well-known-symbols": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
|
||||
|
|
@ -5656,6 +5695,15 @@
|
|||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
|
||||
"dependencies": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/which": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
||||
|
|
@ -6344,6 +6392,16 @@
|
|||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
|
||||
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
|
||||
},
|
||||
"@types/node-fetch": {
|
||||
"version": "2.5.12",
|
||||
"resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
|
||||
"integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"@types/node": "*",
|
||||
"form-data": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"@types/normalize-package-data": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
|
||||
|
|
@ -7918,6 +7976,17 @@
|
|||
"locate-path": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"form-data": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
|
||||
"integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
|
||||
"dev": true,
|
||||
"requires": {
|
||||
"asynckit": "^0.4.0",
|
||||
"combined-stream": "^1.0.8",
|
||||
"mime-types": "^2.1.12"
|
||||
}
|
||||
},
|
||||
"forwarded": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
|
||||
|
|
@ -8747,9 +8816,12 @@
|
|||
"dev": true
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
"version": "2.6.5",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
|
||||
"integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
|
||||
"requires": {
|
||||
"whatwg-url": "^5.0.0"
|
||||
}
|
||||
},
|
||||
"node-gyp-build": {
|
||||
"version": "4.2.3",
|
||||
|
|
@ -9992,6 +10064,11 @@
|
|||
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
|
||||
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
|
||||
},
|
||||
"tr46": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
|
||||
"integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
|
||||
},
|
||||
"trim-off-newlines": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
|
||||
|
|
@ -10220,9 +10297,10 @@
|
|||
}
|
||||
},
|
||||
"typescript": {
|
||||
"version": "4.3.4",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
|
||||
"integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew=="
|
||||
"version": "4.4.4",
|
||||
"resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
|
||||
"integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
|
||||
"peer": true
|
||||
},
|
||||
"uid2": {
|
||||
"version": "0.0.3",
|
||||
|
|
@ -10382,12 +10460,26 @@
|
|||
"defaults": "^1.0.3"
|
||||
}
|
||||
},
|
||||
"webidl-conversions": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
|
||||
"integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
|
||||
},
|
||||
"well-known-symbols": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
|
||||
"integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==",
|
||||
"dev": true
|
||||
},
|
||||
"whatwg-url": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
|
||||
"integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
|
||||
"requires": {
|
||||
"tr46": "~0.0.3",
|
||||
"webidl-conversions": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"which": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
|
||||
|
|
|
|||
|
|
@ -51,6 +51,7 @@
|
|||
"lodash.pick": "^4.4.0",
|
||||
"moment-timezone": "^0.5.21",
|
||||
"mysql": "^2.16.0",
|
||||
"node-fetch": "^2.6.5",
|
||||
"parse-color": "^1.0.0",
|
||||
"passport": "^0.4.0",
|
||||
"passport-custom": "^1.0.5",
|
||||
|
|
@ -81,6 +82,7 @@
|
|||
"@types/lodash.at": "^4.6.3",
|
||||
"@types/moment-timezone": "^0.5.6",
|
||||
"@types/node": "^14.0.14",
|
||||
"@types/node-fetch": "^2.5.12",
|
||||
"@types/passport": "^1.0.0",
|
||||
"@types/passport-oauth2": "^1.4.8",
|
||||
"@types/passport-strategy": "^0.2.35",
|
||||
|
|
@ -90,8 +92,7 @@
|
|||
"ava": "^3.10.0",
|
||||
"rimraf": "^2.6.2",
|
||||
"source-map-support": "^0.5.16",
|
||||
"tsc-watch": "^4.0.0",
|
||||
"typescript": "^4.3.4"
|
||||
"tsc-watch": "^4.0.0"
|
||||
},
|
||||
"ava": {
|
||||
"files": [
|
||||
|
|
|
|||
229
backend/src/data/Phisherman.ts
Normal file
229
backend/src/data/Phisherman.ts
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
import { getRepository, Repository } from "typeorm";
|
||||
import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry";
|
||||
import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman";
|
||||
import fetch, { Headers } from "node-fetch";
|
||||
import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
|
||||
import moment from "moment-timezone";
|
||||
import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
|
||||
import crypto from "crypto";
|
||||
|
||||
const API_URL = "https://api.phisherman.gg";
|
||||
const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
|
||||
|
||||
let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
|
||||
|
||||
const pendingApiRequests: Map<string, Promise<unknown>> = new Map();
|
||||
const pendingDomainInfoChecks: Map<string, Promise<PhishermanDomainInfo | null>> = new Map();
|
||||
|
||||
type MemoryCacheEntry = {
|
||||
info: PhishermanDomainInfo | null;
|
||||
expires: number;
|
||||
};
|
||||
const memoryCache: Map<string, MemoryCacheEntry> = new Map();
|
||||
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of memoryCache.entries()) {
|
||||
if (entry.expires <= now) {
|
||||
memoryCache.delete(key);
|
||||
}
|
||||
}
|
||||
}, 2 * MINUTES);
|
||||
|
||||
const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES;
|
||||
const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES;
|
||||
const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS;
|
||||
|
||||
const KEY_VALIDITY_LIFETIME = 24 * HOURS;
|
||||
|
||||
let cacheRepository: Repository<PhishermanCacheEntry> | null = null;
|
||||
function getCacheRepository(): Repository<PhishermanCacheEntry> {
|
||||
if (cacheRepository == null) {
|
||||
cacheRepository = getRepository(PhishermanCacheEntry);
|
||||
}
|
||||
return cacheRepository;
|
||||
}
|
||||
|
||||
let keyCacheRepository: Repository<PhishermanKeyCacheEntry> | null = null;
|
||||
function getKeyCacheRepository(): Repository<PhishermanKeyCacheEntry> {
|
||||
if (keyCacheRepository == null) {
|
||||
keyCacheRepository = getRepository(PhishermanKeyCacheEntry);
|
||||
}
|
||||
return keyCacheRepository;
|
||||
}
|
||||
|
||||
class PhishermanApiError extends Error {
|
||||
status: number;
|
||||
constructor(status: number, message: string) {
|
||||
super(message);
|
||||
this.status = status;
|
||||
}
|
||||
}
|
||||
|
||||
export function hasPhishermanMasterAPIKey() {
|
||||
return MASTER_API_KEY != null && MASTER_API_KEY !== "";
|
||||
}
|
||||
|
||||
export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean {
|
||||
return info.classification === "safe";
|
||||
}
|
||||
|
||||
const leadingSlashRegex = /^\/+/g;
|
||||
function trimLeadingSlash(str: string): string {
|
||||
return str.replace(leadingSlashRegex, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an arbitrary API call to the Phisherman API
|
||||
*/
|
||||
async function apiCall<T>(
|
||||
method: "GET" | "POST",
|
||||
resource: string,
|
||||
payload?: Record<string, unknown> | null,
|
||||
): Promise<T> {
|
||||
if (!hasPhishermanMasterAPIKey()) {
|
||||
throw new Error("Phisherman master API key missing");
|
||||
}
|
||||
|
||||
const url = `${API_URL}/${trimLeadingSlash(resource)}`;
|
||||
const key = `${method} ${url}`;
|
||||
|
||||
if (pendingApiRequests.has(key)) {
|
||||
return pendingApiRequests.get(key)! as Promise<T>;
|
||||
}
|
||||
|
||||
const requestPromise = (async () => {
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
headers: new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${MASTER_API_KEY}`,
|
||||
}),
|
||||
body: payload ? JSON.stringify(payload) : undefined,
|
||||
});
|
||||
const data = await response.json().catch(() => null);
|
||||
if (!response.ok || (data as any)?.success === false) {
|
||||
throw new PhishermanApiError(response.status, (data as any)?.message ?? "");
|
||||
}
|
||||
return data;
|
||||
})();
|
||||
requestPromise.finally(() => {
|
||||
pendingApiRequests.delete(key);
|
||||
});
|
||||
pendingApiRequests.set(key, requestPromise);
|
||||
return requestPromise as Promise<T>;
|
||||
}
|
||||
|
||||
type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo;
|
||||
async function fetchDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
||||
// tslint:disable-next-line:no-console
|
||||
console.log(`[PHISHERMAN] Requesting domain information: ${domain}`);
|
||||
const result = await apiCall<Record<string, DomainInfoApiCallResult>>("GET", `/v2/domains/info/${domain}`);
|
||||
const domainInfo = result[domain];
|
||||
if (domainInfo.classification === "unknown") {
|
||||
return null;
|
||||
}
|
||||
return domainInfo;
|
||||
}
|
||||
|
||||
export async function getPhishermanDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
||||
if (pendingDomainInfoChecks.has(domain)) {
|
||||
return pendingDomainInfoChecks.get(domain)!;
|
||||
}
|
||||
|
||||
const promise = (async () => {
|
||||
if (memoryCache.has(domain)) {
|
||||
return memoryCache.get(domain)!.info;
|
||||
}
|
||||
|
||||
const dbCache = getCacheRepository();
|
||||
const existingCachedEntry = await dbCache.findOne({ domain });
|
||||
if (existingCachedEntry) {
|
||||
return existingCachedEntry.data;
|
||||
}
|
||||
|
||||
const freshData = await fetchDomainInfo(domain);
|
||||
const expiryTime =
|
||||
freshData === null
|
||||
? UNKNOWN_DOMAIN_CACHE_LIFETIME
|
||||
: phishermanDomainIsSafe(freshData)
|
||||
? SAFE_DOMAIN_CACHE_LIFETIME
|
||||
: DETECTED_DOMAIN_CACHE_LIFETIME;
|
||||
memoryCache.set(domain, {
|
||||
info: freshData,
|
||||
expires: Date.now() + expiryTime,
|
||||
});
|
||||
|
||||
if (freshData) {
|
||||
// Database cache only stores safe/detected domains, not unknown ones
|
||||
await dbCache.insert({
|
||||
domain,
|
||||
data: freshData,
|
||||
expires_at: moment().add(expiryTime, "ms").format(DBDateFormat),
|
||||
});
|
||||
}
|
||||
|
||||
return freshData;
|
||||
})();
|
||||
promise.finally(() => {
|
||||
pendingDomainInfoChecks.delete(domain);
|
||||
});
|
||||
pendingDomainInfoChecks.set(domain, promise);
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
export async function phishermanApiKeyIsValid(apiKey: string): Promise<boolean> {
|
||||
const keyCache = getKeyCacheRepository();
|
||||
const hash = crypto.createHash("sha256").update(apiKey).digest("hex");
|
||||
const entry = await keyCache.findOne({ hash });
|
||||
if (entry) {
|
||||
return entry.is_valid;
|
||||
}
|
||||
|
||||
const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey });
|
||||
|
||||
await keyCache.insert({
|
||||
hash,
|
||||
is_valid: isValid,
|
||||
expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat),
|
||||
});
|
||||
|
||||
return isValid;
|
||||
}
|
||||
|
||||
export function trackPhishermanCaughtDomain(apiKey: string, domain: string) {
|
||||
if (!caughtDomainTrackingMap.has(apiKey)) {
|
||||
caughtDomainTrackingMap.set(apiKey, new Map());
|
||||
}
|
||||
const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!;
|
||||
if (!apiKeyMap.has(domain)) {
|
||||
apiKeyMap.set(domain, []);
|
||||
}
|
||||
const timestamps = apiKeyMap.get(domain)!;
|
||||
timestamps.push(Date.now());
|
||||
}
|
||||
|
||||
export async function reportTrackedDomainsToPhisherman() {
|
||||
const result = {};
|
||||
for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) {
|
||||
result[apiKey] = {};
|
||||
for (const [domain, timestamps] of domains.entries()) {
|
||||
result[apiKey][domain] = timestamps;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(result).length > 0) {
|
||||
await apiCall("POST", "/v2/phish/caught/bulk", result);
|
||||
caughtDomainTrackingMap = new Map();
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteStalePhishermanCacheEntries() {
|
||||
await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
||||
}
|
||||
|
||||
export async function deleteStalePhishermanKeyCacheEntries() {
|
||||
await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
||||
}
|
||||
18
backend/src/data/entities/PhishermanCacheEntry.ts
Normal file
18
backend/src/data/entities/PhishermanCacheEntry.ts
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
import { Column, Entity, PrimaryColumn } from "typeorm";
|
||||
import { PhishermanDomainInfo } from "../types/phisherman";
|
||||
|
||||
@Entity("phisherman_cache")
|
||||
export class PhishermanCacheEntry {
|
||||
@Column()
|
||||
@PrimaryColumn()
|
||||
id: number;
|
||||
|
||||
@Column()
|
||||
domain: string;
|
||||
|
||||
@Column("simple-json")
|
||||
data: PhishermanDomainInfo;
|
||||
|
||||
@Column()
|
||||
expires_at: string;
|
||||
}
|
||||
17
backend/src/data/entities/PhishermanKeyCacheEntry.ts
Normal file
17
backend/src/data/entities/PhishermanKeyCacheEntry.ts
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import { Column, Entity, PrimaryColumn } from "typeorm";
|
||||
|
||||
@Entity("phisherman_key_cache")
|
||||
export class PhishermanKeyCacheEntry {
|
||||
@Column()
|
||||
@PrimaryColumn()
|
||||
id: number;
|
||||
|
||||
@Column()
|
||||
hash: string;
|
||||
|
||||
@Column()
|
||||
is_valid: boolean;
|
||||
|
||||
@Column()
|
||||
expires_at: string;
|
||||
}
|
||||
26
backend/src/data/loops/phishermanLoops.ts
Normal file
26
backend/src/data/loops/phishermanLoops.ts
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
import { HOURS, MINUTES } from "../../utils";
|
||||
import {
|
||||
deleteStalePhishermanCacheEntries,
|
||||
deleteStalePhishermanKeyCacheEntries,
|
||||
reportTrackedDomainsToPhisherman,
|
||||
} from "../Phisherman";
|
||||
|
||||
const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES;
|
||||
const REPORT_LOOP_INTERVAL = 15 * MINUTES;
|
||||
|
||||
export async function runPhishermanCacheCleanupLoop() {
|
||||
console.log("[PHISHERMAN] Deleting stale cache entries");
|
||||
await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err));
|
||||
|
||||
console.log("[PHISHERMAN] Deleting stale key cache entries");
|
||||
await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err));
|
||||
|
||||
setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL);
|
||||
}
|
||||
|
||||
export async function runPhishermanReportingLoop() {
|
||||
console.log("[PHISHERMAN] Reporting tracked domains");
|
||||
await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err));
|
||||
|
||||
setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL);
|
||||
}
|
||||
32
backend/src/data/types/phisherman.ts
Normal file
32
backend/src/data/types/phisherman.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
export interface PhishermanUnknownDomain {
|
||||
classification: "unknown";
|
||||
}
|
||||
|
||||
export interface PhishermanDomainInfo {
|
||||
status: string;
|
||||
lastChecked: string;
|
||||
verifiedPhish: boolean;
|
||||
classification: "safe" | "malicious";
|
||||
created: string;
|
||||
firstSeen: string | null;
|
||||
lastSeen: string | null;
|
||||
targetedBrand: string;
|
||||
phishCaught: number;
|
||||
details: PhishermanDomainInfoDetails;
|
||||
}
|
||||
|
||||
export interface PhishermanDomainInfoDetails {
|
||||
phishTankId: string | null;
|
||||
urlScanId: string;
|
||||
websiteScreenshot: string;
|
||||
ip_address: string;
|
||||
asn: PhishermanDomainInfoAsn;
|
||||
registry: string;
|
||||
country: string;
|
||||
}
|
||||
|
||||
export interface PhishermanDomainInfoAsn {
|
||||
asn: string;
|
||||
asn_name: string;
|
||||
route: string;
|
||||
}
|
||||
|
|
@ -33,6 +33,8 @@ import { runSavedMessageCleanupLoop } from "./data/loops/savedMessageCleanupLoop
|
|||
import { performance } from "perf_hooks";
|
||||
import { setProfiler } from "./profiler";
|
||||
import { enableProfiling } from "./utils/easyProfiler";
|
||||
import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./data/loops/phishermanLoops";
|
||||
import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
|
||||
|
||||
if (!process.env.KEY) {
|
||||
// tslint:disable-next-line:no-console
|
||||
|
|
@ -363,6 +365,13 @@ connect().then(async () => {
|
|||
runExpiredArchiveDeletionLoop();
|
||||
await sleep(10 * SECONDS);
|
||||
runSavedMessageCleanupLoop();
|
||||
|
||||
if (hasPhishermanMasterAPIKey()) {
|
||||
await sleep(10 * SECONDS);
|
||||
runPhishermanCacheCleanupLoop();
|
||||
await sleep(10 * SECONDS);
|
||||
runPhishermanReportingLoop();
|
||||
}
|
||||
});
|
||||
|
||||
setProfiler(bot.profiler);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
import { MigrationInterface, QueryRunner, Table } from "typeorm";
|
||||
|
||||
export class CreatePhishermanCacheTable1634563901575 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.createTable(
|
||||
new Table({
|
||||
name: "phisherman_cache",
|
||||
columns: [
|
||||
{
|
||||
name: "id",
|
||||
type: "int",
|
||||
isPrimary: true,
|
||||
isGenerated: true,
|
||||
generationStrategy: "increment",
|
||||
},
|
||||
{
|
||||
name: "domain",
|
||||
type: "varchar",
|
||||
length: "255",
|
||||
isUnique: true,
|
||||
},
|
||||
{
|
||||
name: "data",
|
||||
type: "text",
|
||||
},
|
||||
{
|
||||
name: "expires_at",
|
||||
type: "datetime",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.dropTable("phisherman_cache");
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
import { MigrationInterface, QueryRunner, Table } from "typeorm";
|
||||
|
||||
export class CreatePhishermanKeyCacheTable1635596150234 implements MigrationInterface {
|
||||
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.createTable(
|
||||
new Table({
|
||||
name: "phisherman_key_cache",
|
||||
columns: [
|
||||
{
|
||||
name: "id",
|
||||
type: "int",
|
||||
isPrimary: true,
|
||||
isGenerated: true,
|
||||
generationStrategy: "increment",
|
||||
},
|
||||
{
|
||||
name: "hash",
|
||||
type: "varchar",
|
||||
length: "255",
|
||||
isUnique: true,
|
||||
},
|
||||
{
|
||||
name: "is_valid",
|
||||
type: "tinyint",
|
||||
},
|
||||
{
|
||||
name: "expires_at",
|
||||
type: "datetime",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||
await queryRunner.dropTable("phisherman_key_cache");
|
||||
}
|
||||
}
|
||||
|
|
@ -30,6 +30,7 @@ import { clearOldRecentSpam } from "./functions/clearOldRecentSpam";
|
|||
import { pluginInfo } from "./info";
|
||||
import { availableTriggers } from "./triggers/availableTriggers";
|
||||
import { AutomodPluginType, ConfigSchema } from "./types";
|
||||
import { PhishermanPlugin } from "../Phisherman/PhishermanPlugin";
|
||||
|
||||
const defaultOptions = {
|
||||
config: {
|
||||
|
|
@ -183,6 +184,7 @@ export const AutomodPlugin = zeppelinGuildPlugin<AutomodPluginType>()({
|
|||
ModActionsPlugin,
|
||||
MutesPlugin,
|
||||
CountersPlugin,
|
||||
PhishermanPlugin,
|
||||
],
|
||||
|
||||
configSchema: ConfigSchema,
|
||||
|
|
|
|||
|
|
@ -9,10 +9,13 @@ import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions
|
|||
import { automodTrigger } from "../helpers";
|
||||
import { mergeRegexes } from "../../../utils/mergeRegexes";
|
||||
import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex";
|
||||
import { PhishermanPlugin } from "../../Phisherman/PhishermanPlugin";
|
||||
import { phishermanDomainIsSafe } from "../../../data/Phisherman";
|
||||
|
||||
interface MatchResultType {
|
||||
type: MatchableTextType;
|
||||
link: string;
|
||||
details?: string;
|
||||
}
|
||||
|
||||
const regexCache = new WeakMap<any, RegExp[]>();
|
||||
|
|
@ -28,6 +31,12 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
|
|||
exclude_words: tNullable(t.array(t.string)),
|
||||
include_regex: tNullable(t.array(TRegex)),
|
||||
exclude_regex: tNullable(t.array(TRegex)),
|
||||
phisherman: tNullable(
|
||||
t.type({
|
||||
include_suspected: tNullable(t.boolean),
|
||||
include_verified: tNullable(t.boolean),
|
||||
}),
|
||||
),
|
||||
only_real_links: t.boolean,
|
||||
match_messages: t.boolean,
|
||||
match_embeds: t.boolean,
|
||||
|
|
@ -150,6 +159,25 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (trigger.phisherman) {
|
||||
const phishermanResult = await pluginData.getPlugin(PhishermanPlugin).getDomainInfo(normalizedHostname);
|
||||
if (phishermanResult != null && !phishermanDomainIsSafe(phishermanResult)) {
|
||||
if (
|
||||
(trigger.phisherman.include_suspected && !phishermanResult.verifiedPhish) ||
|
||||
(trigger.phisherman.include_verified && phishermanResult.verifiedPhish)
|
||||
) {
|
||||
const suspectedVerified = phishermanResult.verifiedPhish ? "verified" : "suspected";
|
||||
return {
|
||||
extra: {
|
||||
type,
|
||||
link: link.input,
|
||||
details: `using Phisherman (${suspectedVerified})`,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -158,6 +186,11 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
|
|||
|
||||
renderMatchInformation({ pluginData, contexts, matchResult }) {
|
||||
const partialSummary = getTextMatchPartialSummary(pluginData, matchResult.extra.type, contexts[0]);
|
||||
return `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\` in ${partialSummary}`;
|
||||
let information = `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\``;
|
||||
if (matchResult.extra.details) {
|
||||
information += ` ${matchResult.extra.details}`;
|
||||
}
|
||||
information += ` in ${partialSummary}`;
|
||||
return information;
|
||||
},
|
||||
});
|
||||
|
|
|
|||
49
backend/src/plugins/Phisherman/PhishermanPlugin.ts
Normal file
49
backend/src/plugins/Phisherman/PhishermanPlugin.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import { PluginOptions, typedGuildCommand } from "knub";
|
||||
import { GuildPingableRoles } from "../../data/GuildPingableRoles";
|
||||
import { zeppelinGuildPlugin } from "../ZeppelinPluginBlueprint";
|
||||
import { ConfigSchema, PhishermanPluginType } from "./types";
|
||||
import {
|
||||
getPhishermanDomainInfo,
|
||||
hasPhishermanMasterAPIKey,
|
||||
phishermanApiKeyIsValid,
|
||||
reportTrackedDomainsToPhisherman,
|
||||
} from "../../data/Phisherman";
|
||||
import { mapToPublicFn } from "../../pluginUtils";
|
||||
import { getDomainInfo } from "./functions/getDomainInfo";
|
||||
import { pluginInfo } from "./info";
|
||||
|
||||
const defaultOptions: PluginOptions<PhishermanPluginType> = {
|
||||
config: {
|
||||
api_key: null,
|
||||
},
|
||||
overrides: [],
|
||||
};
|
||||
|
||||
export const PhishermanPlugin = zeppelinGuildPlugin<PhishermanPluginType>()({
|
||||
name: "phisherman",
|
||||
showInDocs: true,
|
||||
info: pluginInfo,
|
||||
|
||||
configSchema: ConfigSchema,
|
||||
defaultOptions,
|
||||
|
||||
// prettier-ignore
|
||||
public: {
|
||||
getDomainInfo: mapToPublicFn(getDomainInfo),
|
||||
},
|
||||
|
||||
async beforeLoad(pluginData) {
|
||||
pluginData.state.validApiKey = null;
|
||||
|
||||
if (!hasPhishermanMasterAPIKey()) {
|
||||
// tslint:disable-next-line:no-console
|
||||
console.warn("Could not load Phisherman plugin: master API key is missing");
|
||||
return;
|
||||
}
|
||||
|
||||
const apiKey = pluginData.config.get().api_key;
|
||||
if (apiKey && (await phishermanApiKeyIsValid(apiKey).catch(() => false))) {
|
||||
pluginData.state.validApiKey = apiKey;
|
||||
}
|
||||
},
|
||||
});
|
||||
20
backend/src/plugins/Phisherman/functions/getDomainInfo.ts
Normal file
20
backend/src/plugins/Phisherman/functions/getDomainInfo.ts
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
import { GuildPluginData } from "knub";
|
||||
import { PhishermanPluginType } from "../types";
|
||||
import { PhishermanDomainInfo } from "../../../data/types/phisherman";
|
||||
import { getPhishermanDomainInfo, phishermanDomainIsSafe, trackPhishermanCaughtDomain } from "../../../data/Phisherman";
|
||||
|
||||
export async function getDomainInfo(
|
||||
pluginData: GuildPluginData<PhishermanPluginType>,
|
||||
domain: string,
|
||||
): Promise<PhishermanDomainInfo | null> {
|
||||
if (!pluginData.state.validApiKey) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const info = await getPhishermanDomainInfo(domain).catch(() => null);
|
||||
if (info != null && !phishermanDomainIsSafe(info)) {
|
||||
trackPhishermanCaughtDomain(pluginData.state.validApiKey, domain);
|
||||
}
|
||||
|
||||
return info;
|
||||
}
|
||||
41
backend/src/plugins/Phisherman/info.ts
Normal file
41
backend/src/plugins/Phisherman/info.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { trimPluginDescription } from "../../utils";
|
||||
import { ZeppelinGuildPluginBlueprint } from "../ZeppelinPluginBlueprint";
|
||||
|
||||
export const pluginInfo: ZeppelinGuildPluginBlueprint["info"] = {
|
||||
prettyName: "Phisherman",
|
||||
description: trimPluginDescription(`
|
||||
Match scam/phishing links using the Phisherman API. See https://phisherman.gg/ for more details!
|
||||
`),
|
||||
configurationGuide: trimPluginDescription(`
|
||||
### Getting started
|
||||
To get started, request an API key for Phisherman following the instructions at https://docs.phisherman.gg/#/api/getting-started?id=requesting-api-access.
|
||||
Then, add the api key to the plugin's config:
|
||||
|
||||
~~~yml
|
||||
phisherman:
|
||||
config:
|
||||
api_key: "your key here"
|
||||
~~~
|
||||
|
||||
### Note
|
||||
When using Phisherman features in Zeppelin, Zeppelin reports statistics about checked links back to Phisherman. This only includes the domain (e.g. zeppelin.gg), not the full link.
|
||||
|
||||
### Usage with Automod
|
||||
Once you have configured the Phisherman plugin, you are ready to use it with automod. Currently, Phisherman is available as an option in the \`match_links\` plugin:
|
||||
|
||||
~~~yml
|
||||
automod:
|
||||
config:
|
||||
rules:
|
||||
# Clean any scam links detected by Phisherman
|
||||
filter_scam_links:
|
||||
triggers:
|
||||
- match_links:
|
||||
phisherman:
|
||||
include_suspected: true # It's recommended to keep this enabled to catch new scam domains quickly
|
||||
include_verified: true
|
||||
actions:
|
||||
clean: true
|
||||
~~~
|
||||
`),
|
||||
};
|
||||
16
backend/src/plugins/Phisherman/types.ts
Normal file
16
backend/src/plugins/Phisherman/types.ts
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
import * as t from "io-ts";
|
||||
import { BasePluginType } from "knub";
|
||||
import { tNullable } from "../../utils";
|
||||
|
||||
export const ConfigSchema = t.type({
|
||||
api_key: tNullable(t.string),
|
||||
});
|
||||
export type TConfigSchema = t.TypeOf<typeof ConfigSchema>;
|
||||
|
||||
export interface PhishermanPluginType extends BasePluginType {
|
||||
config: TConfigSchema;
|
||||
|
||||
state: {
|
||||
validApiKey: string | null;
|
||||
};
|
||||
}
|
||||
|
|
@ -34,6 +34,7 @@ import { UsernameSaverPlugin } from "./UsernameSaver/UsernameSaverPlugin";
|
|||
import { UtilityPlugin } from "./Utility/UtilityPlugin";
|
||||
import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin";
|
||||
import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint";
|
||||
import { PhishermanPlugin } from "./Phisherman/PhishermanPlugin";
|
||||
|
||||
// prettier-ignore
|
||||
export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
|
||||
|
|
@ -69,6 +70,7 @@ export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
|
|||
TimeAndDatePlugin,
|
||||
CountersPlugin,
|
||||
ContextMenuPlugin,
|
||||
PhishermanPlugin,
|
||||
];
|
||||
|
||||
// prettier-ignore
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue