From 13c94a81cc5f91c05f2fcdc692d31b8cde192244 Mon Sep 17 00:00:00 2001
From: Dragory <2606411+Dragory@users.noreply.github.com>
Date: Sun, 31 Oct 2021 17:17:31 +0200
Subject: [PATCH] feat: Phisherman integration
---
backend/bot.env.example | 1 +
backend/package-lock.json | 120 +++++++--
backend/package.json | 5 +-
backend/src/data/Phisherman.ts | 229 ++++++++++++++++++
.../src/data/entities/PhishermanCacheEntry.ts | 18 ++
.../data/entities/PhishermanKeyCacheEntry.ts | 17 ++
backend/src/data/loops/phishermanLoops.ts | 26 ++
backend/src/data/types/phisherman.ts | 32 +++
backend/src/index.ts | 9 +
...634563901575-CreatePhishermanCacheTable.ts | 38 +++
...596150234-CreatePhishermanKeyCacheTable.ts | 38 +++
backend/src/plugins/Automod/AutomodPlugin.ts | 2 +
.../plugins/Automod/triggers/matchLinks.ts | 35 ++-
.../plugins/Phisherman/PhishermanPlugin.ts | 49 ++++
.../Phisherman/functions/getDomainInfo.ts | 20 ++
backend/src/plugins/Phisherman/info.ts | 41 ++++
backend/src/plugins/Phisherman/types.ts | 16 ++
backend/src/plugins/availablePlugins.ts | 2 +
18 files changed, 681 insertions(+), 17 deletions(-)
create mode 100644 backend/src/data/Phisherman.ts
create mode 100644 backend/src/data/entities/PhishermanCacheEntry.ts
create mode 100644 backend/src/data/entities/PhishermanKeyCacheEntry.ts
create mode 100644 backend/src/data/loops/phishermanLoops.ts
create mode 100644 backend/src/data/types/phisherman.ts
create mode 100644 backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts
create mode 100644 backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts
create mode 100644 backend/src/plugins/Phisherman/PhishermanPlugin.ts
create mode 100644 backend/src/plugins/Phisherman/functions/getDomainInfo.ts
create mode 100644 backend/src/plugins/Phisherman/info.ts
create mode 100644 backend/src/plugins/Phisherman/types.ts
diff --git a/backend/bot.env.example b/backend/bot.env.example
index 7fdd7e5c..b701f4ed 100644
--- a/backend/bot.env.example
+++ b/backend/bot.env.example
@@ -4,3 +4,4 @@ DB_USER=
DB_PASSWORD=
DB_DATABASE=
PROFILING=false
+PHISHERMAN_API_KEY=
diff --git a/backend/package-lock.json b/backend/package-lock.json
index cdc6456a..a5e5cf8b 100644
--- a/backend/package-lock.json
+++ b/backend/package-lock.json
@@ -36,6 +36,7 @@
"lodash.pick": "^4.4.0",
"moment-timezone": "^0.5.21",
"mysql": "^2.16.0",
+ "node-fetch": "^2.6.5",
"parse-color": "^1.0.0",
"passport": "^0.4.0",
"passport-custom": "^1.0.5",
@@ -66,6 +67,7 @@
"@types/lodash.at": "^4.6.3",
"@types/moment-timezone": "^0.5.6",
"@types/node": "^14.0.14",
+ "@types/node-fetch": "^2.5.12",
"@types/passport": "^1.0.0",
"@types/passport-oauth2": "^1.4.8",
"@types/passport-strategy": "^0.2.35",
@@ -75,8 +77,7 @@
"ava": "^3.10.0",
"rimraf": "^2.6.2",
"source-map-support": "^0.5.16",
- "tsc-watch": "^4.0.0",
- "typescript": "^4.3.4"
+ "tsc-watch": "^4.0.0"
}
},
"../../Knub": {
@@ -424,6 +425,16 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
},
+ "node_modules/@types/node-fetch": {
+ "version": "2.5.12",
+ "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
+ "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
+ "dev": true,
+ "dependencies": {
+ "@types/node": "*",
+ "form-data": "^3.0.0"
+ }
+ },
"node_modules/@types/normalize-package-data": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
@@ -2436,6 +2447,20 @@
"node": ">=6"
}
},
+ "node_modules/form-data": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+ "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+ "dev": true,
+ "dependencies": {
+ "asynckit": "^0.4.0",
+ "combined-stream": "^1.0.8",
+ "mime-types": "^2.1.12"
+ },
+ "engines": {
+ "node": ">= 6"
+ }
+ },
"node_modules/forwarded": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
@@ -3513,9 +3538,12 @@
"dev": true
},
"node_modules/node-fetch": {
- "version": "2.6.1",
- "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
- "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
+ "version": "2.6.5",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
+ "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
+ "dependencies": {
+ "whatwg-url": "^5.0.0"
+ },
"engines": {
"node": "4.x || >=6.0.0"
}
@@ -5127,6 +5155,11 @@
"node": ">=0.6"
}
},
+ "node_modules/tr46": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+ "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+ },
"node_modules/trim-off-newlines": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
@@ -5430,9 +5463,10 @@
}
},
"node_modules/typescript": {
- "version": "4.3.4",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
- "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==",
+ "version": "4.4.4",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
+ "integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
+ "peer": true,
"bin": {
"tsc": "bin/tsc",
"tsserver": "bin/tsserver"
@@ -5647,6 +5681,11 @@
"defaults": "^1.0.3"
}
},
+ "node_modules/webidl-conversions": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+ "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+ },
"node_modules/well-known-symbols": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
@@ -5656,6 +5695,15 @@
"node": ">=6"
}
},
+ "node_modules/whatwg-url": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+ "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+ "dependencies": {
+ "tr46": "~0.0.3",
+ "webidl-conversions": "^3.0.0"
+ }
+ },
"node_modules/which": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
@@ -6344,6 +6392,16 @@
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
"integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
},
+ "@types/node-fetch": {
+ "version": "2.5.12",
+ "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
+ "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
+ "dev": true,
+ "requires": {
+ "@types/node": "*",
+ "form-data": "^3.0.0"
+ }
+ },
"@types/normalize-package-data": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
@@ -7918,6 +7976,17 @@
"locate-path": "^3.0.0"
}
},
+ "form-data": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+ "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+ "dev": true,
+ "requires": {
+ "asynckit": "^0.4.0",
+ "combined-stream": "^1.0.8",
+ "mime-types": "^2.1.12"
+ }
+ },
"forwarded": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
@@ -8747,9 +8816,12 @@
"dev": true
},
"node-fetch": {
- "version": "2.6.1",
- "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
- "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
+ "version": "2.6.5",
+ "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
+ "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
+ "requires": {
+ "whatwg-url": "^5.0.0"
+ }
},
"node-gyp-build": {
"version": "4.2.3",
@@ -9992,6 +10064,11 @@
"resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
"integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
},
+ "tr46": {
+ "version": "0.0.3",
+ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+ "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+ },
"trim-off-newlines": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
@@ -10220,9 +10297,10 @@
}
},
"typescript": {
- "version": "4.3.4",
- "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
- "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew=="
+ "version": "4.4.4",
+ "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
+ "integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
+ "peer": true
},
"uid2": {
"version": "0.0.3",
@@ -10382,12 +10460,26 @@
"defaults": "^1.0.3"
}
},
+ "webidl-conversions": {
+ "version": "3.0.1",
+ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+ "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+ },
"well-known-symbols": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
"integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==",
"dev": true
},
+ "whatwg-url": {
+ "version": "5.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+ "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+ "requires": {
+ "tr46": "~0.0.3",
+ "webidl-conversions": "^3.0.0"
+ }
+ },
"which": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
diff --git a/backend/package.json b/backend/package.json
index a9870eec..1a977808 100644
--- a/backend/package.json
+++ b/backend/package.json
@@ -51,6 +51,7 @@
"lodash.pick": "^4.4.0",
"moment-timezone": "^0.5.21",
"mysql": "^2.16.0",
+ "node-fetch": "^2.6.5",
"parse-color": "^1.0.0",
"passport": "^0.4.0",
"passport-custom": "^1.0.5",
@@ -81,6 +82,7 @@
"@types/lodash.at": "^4.6.3",
"@types/moment-timezone": "^0.5.6",
"@types/node": "^14.0.14",
+ "@types/node-fetch": "^2.5.12",
"@types/passport": "^1.0.0",
"@types/passport-oauth2": "^1.4.8",
"@types/passport-strategy": "^0.2.35",
@@ -90,8 +92,7 @@
"ava": "^3.10.0",
"rimraf": "^2.6.2",
"source-map-support": "^0.5.16",
- "tsc-watch": "^4.0.0",
- "typescript": "^4.3.4"
+ "tsc-watch": "^4.0.0"
},
"ava": {
"files": [
diff --git a/backend/src/data/Phisherman.ts b/backend/src/data/Phisherman.ts
new file mode 100644
index 00000000..25fb59ce
--- /dev/null
+++ b/backend/src/data/Phisherman.ts
@@ -0,0 +1,229 @@
+import { getRepository, Repository } from "typeorm";
+import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry";
+import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman";
+import fetch, { Headers } from "node-fetch";
+import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
+import moment from "moment-timezone";
+import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
+import crypto from "crypto";
+
+const API_URL = "https://api.phisherman.gg";
+const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
+
+let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
+
+const pendingApiRequests: Map<string, Promise<unknown>> = new Map();
+const pendingDomainInfoChecks: Map<string, Promise<PhishermanDomainInfo | null>> = new Map();
+
+type MemoryCacheEntry = {
+ info: PhishermanDomainInfo | null;
+ expires: number;
+};
+const memoryCache: Map<string, MemoryCacheEntry> = new Map();
+
+setInterval(() => {
+ const now = Date.now();
+ for (const [key, entry] of memoryCache.entries()) {
+ if (entry.expires <= now) {
+ memoryCache.delete(key);
+ }
+ }
+}, 2 * MINUTES);
+
+const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES;
+const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES;
+const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS;
+
+const KEY_VALIDITY_LIFETIME = 24 * HOURS;
+
+let cacheRepository: Repository<PhishermanCacheEntry> | null = null;
+function getCacheRepository(): Repository<PhishermanCacheEntry> {
+ if (cacheRepository == null) {
+ cacheRepository = getRepository(PhishermanCacheEntry);
+ }
+ return cacheRepository;
+}
+
+let keyCacheRepository: Repository<PhishermanKeyCacheEntry> | null = null;
+function getKeyCacheRepository(): Repository<PhishermanKeyCacheEntry> {
+ if (keyCacheRepository == null) {
+ keyCacheRepository = getRepository(PhishermanKeyCacheEntry);
+ }
+ return keyCacheRepository;
+}
+
+class PhishermanApiError extends Error {
+ status: number;
+ constructor(status: number, message: string) {
+ super(message);
+ this.status = status;
+ }
+}
+
+export function hasPhishermanMasterAPIKey() {
+ return MASTER_API_KEY != null && MASTER_API_KEY !== "";
+}
+
+export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean {
+ return info.classification === "safe";
+}
+
+const leadingSlashRegex = /^\/+/g;
+function trimLeadingSlash(str: string): string {
+ return str.replace(leadingSlashRegex, "");
+}
+
+/**
+ * Make an arbitrary API call to the Phisherman API
+ */
+async function apiCall<T>(
+ method: "GET" | "POST",
+ resource: string,
+ payload?: Record<string, unknown> | null,
+): Promise<T> {
+ if (!hasPhishermanMasterAPIKey()) {
+ throw new Error("Phisherman master API key missing");
+ }
+
+ const url = `${API_URL}/${trimLeadingSlash(resource)}`;
+ const key = `${method} ${url}`;
+
+ if (pendingApiRequests.has(key)) {
+ return pendingApiRequests.get(key)! as Promise<T>;
+ }
+
+ const requestPromise = (async () => {
+ const response = await fetch(url, {
+ method,
+ headers: new Headers({
+ Accept: "application/json",
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${MASTER_API_KEY}`,
+ }),
+ body: payload ? JSON.stringify(payload) : undefined,
+ });
+ const data = await response.json().catch(() => null);
+ if (!response.ok || (data as any)?.success === false) {
+ throw new PhishermanApiError(response.status, (data as any)?.message ?? "");
+ }
+ return data;
+ })();
+ requestPromise.finally(() => {
+ pendingApiRequests.delete(key);
+ });
+ pendingApiRequests.set(key, requestPromise);
+ return requestPromise as Promise<T>;
+}
+
+type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo;
+async function fetchDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
+ // tslint:disable-next-line:no-console
+ console.log(`[PHISHERMAN] Requesting domain information: ${domain}`);
+ const result = await apiCall<Record<string, DomainInfoApiCallResult>>("GET", `/v2/domains/info/${domain}`);
+ const domainInfo = result[domain];
+ if (domainInfo.classification === "unknown") {
+ return null;
+ }
+ return domainInfo;
+}
+
+export async function getPhishermanDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
+ if (pendingDomainInfoChecks.has(domain)) {
+ return pendingDomainInfoChecks.get(domain)!;
+ }
+
+ const promise = (async () => {
+ if (memoryCache.has(domain)) {
+ return memoryCache.get(domain)!.info;
+ }
+
+ const dbCache = getCacheRepository();
+ const existingCachedEntry = await dbCache.findOne({ domain });
+ if (existingCachedEntry) {
+ return existingCachedEntry.data;
+ }
+
+ const freshData = await fetchDomainInfo(domain);
+ const expiryTime =
+ freshData === null
+ ? UNKNOWN_DOMAIN_CACHE_LIFETIME
+ : phishermanDomainIsSafe(freshData)
+ ? SAFE_DOMAIN_CACHE_LIFETIME
+ : DETECTED_DOMAIN_CACHE_LIFETIME;
+ memoryCache.set(domain, {
+ info: freshData,
+ expires: Date.now() + expiryTime,
+ });
+
+ if (freshData) {
+ // Database cache only stores safe/detected domains, not unknown ones
+ await dbCache.insert({
+ domain,
+ data: freshData,
+ expires_at: moment().add(expiryTime, "ms").format(DBDateFormat),
+ });
+ }
+
+ return freshData;
+ })();
+ promise.finally(() => {
+ pendingDomainInfoChecks.delete(domain);
+ });
+ pendingDomainInfoChecks.set(domain, promise);
+
+ return promise;
+}
+
+export async function phishermanApiKeyIsValid(apiKey: string): Promise<boolean> {
+ const keyCache = getKeyCacheRepository();
+ const hash = crypto.createHash("sha256").update(apiKey).digest("hex");
+ const entry = await keyCache.findOne({ hash });
+ if (entry) {
+ return entry.is_valid;
+ }
+
+ const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey });
+
+ await keyCache.insert({
+ hash,
+ is_valid: isValid,
+ expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat),
+ });
+
+ return isValid;
+}
+
+export function trackPhishermanCaughtDomain(apiKey: string, domain: string) {
+ if (!caughtDomainTrackingMap.has(apiKey)) {
+ caughtDomainTrackingMap.set(apiKey, new Map());
+ }
+ const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!;
+ if (!apiKeyMap.has(domain)) {
+ apiKeyMap.set(domain, []);
+ }
+ const timestamps = apiKeyMap.get(domain)!;
+ timestamps.push(Date.now());
+}
+
+export async function reportTrackedDomainsToPhisherman() {
+ const result = {};
+ for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) {
+ result[apiKey] = {};
+ for (const [domain, timestamps] of domains.entries()) {
+ result[apiKey][domain] = timestamps;
+ }
+ }
+
+ if (Object.keys(result).length > 0) {
+ await apiCall("POST", "/v2/phish/caught/bulk", result);
+ caughtDomainTrackingMap = new Map();
+ }
+}
+
+export async function deleteStalePhishermanCacheEntries() {
+ await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
+}
+
+export async function deleteStalePhishermanKeyCacheEntries() {
+ await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
+}
diff --git a/backend/src/data/entities/PhishermanCacheEntry.ts b/backend/src/data/entities/PhishermanCacheEntry.ts
new file mode 100644
index 00000000..cad135bf
--- /dev/null
+++ b/backend/src/data/entities/PhishermanCacheEntry.ts
@@ -0,0 +1,18 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+import { PhishermanDomainInfo } from "../types/phisherman";
+
+@Entity("phisherman_cache")
+export class PhishermanCacheEntry {
+ @Column()
+ @PrimaryColumn()
+ id: number;
+
+ @Column()
+ domain: string;
+
+ @Column("simple-json")
+ data: PhishermanDomainInfo;
+
+ @Column()
+ expires_at: string;
+}
diff --git a/backend/src/data/entities/PhishermanKeyCacheEntry.ts b/backend/src/data/entities/PhishermanKeyCacheEntry.ts
new file mode 100644
index 00000000..c4286d1c
--- /dev/null
+++ b/backend/src/data/entities/PhishermanKeyCacheEntry.ts
@@ -0,0 +1,17 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+
+@Entity("phisherman_key_cache")
+export class PhishermanKeyCacheEntry {
+ @Column()
+ @PrimaryColumn()
+ id: number;
+
+ @Column()
+ hash: string;
+
+ @Column()
+ is_valid: boolean;
+
+ @Column()
+ expires_at: string;
+}
diff --git a/backend/src/data/loops/phishermanLoops.ts b/backend/src/data/loops/phishermanLoops.ts
new file mode 100644
index 00000000..507f2166
--- /dev/null
+++ b/backend/src/data/loops/phishermanLoops.ts
@@ -0,0 +1,26 @@
+import { HOURS, MINUTES } from "../../utils";
+import {
+ deleteStalePhishermanCacheEntries,
+ deleteStalePhishermanKeyCacheEntries,
+ reportTrackedDomainsToPhisherman,
+} from "../Phisherman";
+
+const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES;
+const REPORT_LOOP_INTERVAL = 15 * MINUTES;
+
+export async function runPhishermanCacheCleanupLoop() {
+ console.log("[PHISHERMAN] Deleting stale cache entries");
+ await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err));
+
+ console.log("[PHISHERMAN] Deleting stale key cache entries");
+ await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err));
+
+ setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL);
+}
+
+export async function runPhishermanReportingLoop() {
+ console.log("[PHISHERMAN] Reporting tracked domains");
+ await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err));
+
+ setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL);
+}
diff --git a/backend/src/data/types/phisherman.ts b/backend/src/data/types/phisherman.ts
new file mode 100644
index 00000000..418fd43a
--- /dev/null
+++ b/backend/src/data/types/phisherman.ts
@@ -0,0 +1,32 @@
+export interface PhishermanUnknownDomain {
+ classification: "unknown";
+}
+
+export interface PhishermanDomainInfo {
+ status: string;
+ lastChecked: string;
+ verifiedPhish: boolean;
+ classification: "safe" | "malicious";
+ created: string;
+ firstSeen: string | null;
+ lastSeen: string | null;
+ targetedBrand: string;
+ phishCaught: number;
+ details: PhishermanDomainInfoDetails;
+}
+
+export interface PhishermanDomainInfoDetails {
+ phishTankId: string | null;
+ urlScanId: string;
+ websiteScreenshot: string;
+ ip_address: string;
+ asn: PhishermanDomainInfoAsn;
+ registry: string;
+ country: string;
+}
+
+export interface PhishermanDomainInfoAsn {
+ asn: string;
+ asn_name: string;
+ route: string;
+}
diff --git a/backend/src/index.ts b/backend/src/index.ts
index bbb4770b..587a515f 100644
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -33,6 +33,8 @@ import { runSavedMessageCleanupLoop } from "./data/loops/savedMessageCleanupLoop
import { performance } from "perf_hooks";
import { setProfiler } from "./profiler";
import { enableProfiling } from "./utils/easyProfiler";
+import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./data/loops/phishermanLoops";
+import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
if (!process.env.KEY) {
// tslint:disable-next-line:no-console
@@ -363,6 +365,13 @@ connect().then(async () => {
runExpiredArchiveDeletionLoop();
await sleep(10 * SECONDS);
runSavedMessageCleanupLoop();
+
+ if (hasPhishermanMasterAPIKey()) {
+ await sleep(10 * SECONDS);
+ runPhishermanCacheCleanupLoop();
+ await sleep(10 * SECONDS);
+ runPhishermanReportingLoop();
+ }
});
setProfiler(bot.profiler);
diff --git a/backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts b/backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts
new file mode 100644
index 00000000..a0f5543e
--- /dev/null
+++ b/backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts
@@ -0,0 +1,38 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreatePhishermanCacheTable1634563901575 implements MigrationInterface {
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.createTable(
+ new Table({
+ name: "phisherman_cache",
+ columns: [
+ {
+ name: "id",
+ type: "int",
+ isPrimary: true,
+ isGenerated: true,
+ generationStrategy: "increment",
+ },
+ {
+ name: "domain",
+ type: "varchar",
+ length: "255",
+ isUnique: true,
+ },
+ {
+ name: "data",
+ type: "text",
+ },
+ {
+ name: "expires_at",
+ type: "datetime",
+ },
+ ],
+ }),
+ );
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.dropTable("phisherman_cache");
+ }
+}
diff --git a/backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts b/backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts
new file mode 100644
index 00000000..2e9bb7c7
--- /dev/null
+++ b/backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts
@@ -0,0 +1,38 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreatePhishermanKeyCacheTable1635596150234 implements MigrationInterface {
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.createTable(
+ new Table({
+ name: "phisherman_key_cache",
+ columns: [
+ {
+ name: "id",
+ type: "int",
+ isPrimary: true,
+ isGenerated: true,
+ generationStrategy: "increment",
+ },
+ {
+ name: "hash",
+ type: "varchar",
+ length: "255",
+ isUnique: true,
+ },
+ {
+ name: "is_valid",
+ type: "tinyint",
+ },
+ {
+ name: "expires_at",
+ type: "datetime",
+ },
+ ],
+ }),
+ );
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ await queryRunner.dropTable("phisherman_key_cache");
+ }
+}
diff --git a/backend/src/plugins/Automod/AutomodPlugin.ts b/backend/src/plugins/Automod/AutomodPlugin.ts
index 602db445..16339309 100644
--- a/backend/src/plugins/Automod/AutomodPlugin.ts
+++ b/backend/src/plugins/Automod/AutomodPlugin.ts
@@ -30,6 +30,7 @@ import { clearOldRecentSpam } from "./functions/clearOldRecentSpam";
import { pluginInfo } from "./info";
import { availableTriggers } from "./triggers/availableTriggers";
import { AutomodPluginType, ConfigSchema } from "./types";
+import { PhishermanPlugin } from "../Phisherman/PhishermanPlugin";
const defaultOptions = {
config: {
@@ -183,6 +184,7 @@ export const AutomodPlugin = zeppelinGuildPlugin<AutomodPluginType>()({
ModActionsPlugin,
MutesPlugin,
CountersPlugin,
+ PhishermanPlugin,
],
configSchema: ConfigSchema,
diff --git a/backend/src/plugins/Automod/triggers/matchLinks.ts b/backend/src/plugins/Automod/triggers/matchLinks.ts
index 31aa8006..248a19f3 100644
--- a/backend/src/plugins/Automod/triggers/matchLinks.ts
+++ b/backend/src/plugins/Automod/triggers/matchLinks.ts
@@ -9,10 +9,13 @@ import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions
import { automodTrigger } from "../helpers";
import { mergeRegexes } from "../../../utils/mergeRegexes";
import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex";
+import { PhishermanPlugin } from "../../Phisherman/PhishermanPlugin";
+import { phishermanDomainIsSafe } from "../../../data/Phisherman";
interface MatchResultType {
type: MatchableTextType;
link: string;
+ details?: string;
}
const regexCache = new WeakMap<any, RegExp[]>();
@@ -28,6 +31,12 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
exclude_words: tNullable(t.array(t.string)),
include_regex: tNullable(t.array(TRegex)),
exclude_regex: tNullable(t.array(TRegex)),
+ phisherman: tNullable(
+ t.type({
+ include_suspected: tNullable(t.boolean),
+ include_verified: tNullable(t.boolean),
+ }),
+ ),
only_real_links: t.boolean,
match_messages: t.boolean,
match_embeds: t.boolean,
@@ -150,6 +159,25 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
}
}
}
+
+ if (trigger.phisherman) {
+ const phishermanResult = await pluginData.getPlugin(PhishermanPlugin).getDomainInfo(normalizedHostname);
+ if (phishermanResult != null && !phishermanDomainIsSafe(phishermanResult)) {
+ if (
+ (trigger.phisherman.include_suspected && !phishermanResult.verifiedPhish) ||
+ (trigger.phisherman.include_verified && phishermanResult.verifiedPhish)
+ ) {
+ const suspectedVerified = phishermanResult.verifiedPhish ? "verified" : "suspected";
+ return {
+ extra: {
+ type,
+ link: link.input,
+ details: `using Phisherman (${suspectedVerified})`,
+ },
+ };
+ }
+ }
+ }
}
}
@@ -158,6 +186,11 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
renderMatchInformation({ pluginData, contexts, matchResult }) {
const partialSummary = getTextMatchPartialSummary(pluginData, matchResult.extra.type, contexts[0]);
- return `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\` in ${partialSummary}`;
+ let information = `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\``;
+ if (matchResult.extra.details) {
+ information += ` ${matchResult.extra.details}`;
+ }
+ information += ` in ${partialSummary}`;
+ return information;
},
});
diff --git a/backend/src/plugins/Phisherman/PhishermanPlugin.ts b/backend/src/plugins/Phisherman/PhishermanPlugin.ts
new file mode 100644
index 00000000..50a6c321
--- /dev/null
+++ b/backend/src/plugins/Phisherman/PhishermanPlugin.ts
@@ -0,0 +1,49 @@
+import { PluginOptions, typedGuildCommand } from "knub";
+import { GuildPingableRoles } from "../../data/GuildPingableRoles";
+import { zeppelinGuildPlugin } from "../ZeppelinPluginBlueprint";
+import { ConfigSchema, PhishermanPluginType } from "./types";
+import {
+ getPhishermanDomainInfo,
+ hasPhishermanMasterAPIKey,
+ phishermanApiKeyIsValid,
+ reportTrackedDomainsToPhisherman,
+} from "../../data/Phisherman";
+import { mapToPublicFn } from "../../pluginUtils";
+import { getDomainInfo } from "./functions/getDomainInfo";
+import { pluginInfo } from "./info";
+
+const defaultOptions: PluginOptions<PhishermanPluginType> = {
+ config: {
+ api_key: null,
+ },
+ overrides: [],
+};
+
+export const PhishermanPlugin = zeppelinGuildPlugin<PhishermanPluginType>()({
+ name: "phisherman",
+ showInDocs: true,
+ info: pluginInfo,
+
+ configSchema: ConfigSchema,
+ defaultOptions,
+
+ // prettier-ignore
+ public: {
+ getDomainInfo: mapToPublicFn(getDomainInfo),
+ },
+
+ async beforeLoad(pluginData) {
+ pluginData.state.validApiKey = null;
+
+ if (!hasPhishermanMasterAPIKey()) {
+ // tslint:disable-next-line:no-console
+ console.warn("Could not load Phisherman plugin: master API key is missing");
+ return;
+ }
+
+ const apiKey = pluginData.config.get().api_key;
+ if (apiKey && (await phishermanApiKeyIsValid(apiKey).catch(() => false))) {
+ pluginData.state.validApiKey = apiKey;
+ }
+ },
+});
diff --git a/backend/src/plugins/Phisherman/functions/getDomainInfo.ts b/backend/src/plugins/Phisherman/functions/getDomainInfo.ts
new file mode 100644
index 00000000..d5b9cbf3
--- /dev/null
+++ b/backend/src/plugins/Phisherman/functions/getDomainInfo.ts
@@ -0,0 +1,20 @@
+import { GuildPluginData } from "knub";
+import { PhishermanPluginType } from "../types";
+import { PhishermanDomainInfo } from "../../../data/types/phisherman";
+import { getPhishermanDomainInfo, phishermanDomainIsSafe, trackPhishermanCaughtDomain } from "../../../data/Phisherman";
+
+export async function getDomainInfo(
+ pluginData: GuildPluginData<PhishermanPluginType>,
+ domain: string,
+): Promise<PhishermanDomainInfo | null> {
+ if (!pluginData.state.validApiKey) {
+ return null;
+ }
+
+ const info = await getPhishermanDomainInfo(domain).catch(() => null);
+ if (info != null && !phishermanDomainIsSafe(info)) {
+ trackPhishermanCaughtDomain(pluginData.state.validApiKey, domain);
+ }
+
+ return info;
+}
diff --git a/backend/src/plugins/Phisherman/info.ts b/backend/src/plugins/Phisherman/info.ts
new file mode 100644
index 00000000..5c9a1818
--- /dev/null
+++ b/backend/src/plugins/Phisherman/info.ts
@@ -0,0 +1,41 @@
+import { trimPluginDescription } from "../../utils";
+import { ZeppelinGuildPluginBlueprint } from "../ZeppelinPluginBlueprint";
+
+export const pluginInfo: ZeppelinGuildPluginBlueprint["info"] = {
+ prettyName: "Phisherman",
+ description: trimPluginDescription(`
+ Match scam/phishing links using the Phisherman API. See https://phisherman.gg/ for more details!
+ `),
+ configurationGuide: trimPluginDescription(`
+ ### Getting started
+ To get started, request an API key for Phisherman following the instructions at https://docs.phisherman.gg/#/api/getting-started?id=requesting-api-access.
+ Then, add the api key to the plugin's config:
+
+ ~~~yml
+ phisherman:
+ config:
+ api_key: "your key here"
+ ~~~
+
+ ### Note
+ When using Phisherman features in Zeppelin, Zeppelin reports statistics about checked links back to Phisherman. This only includes the domain (e.g. zeppelin.gg), not the full link.
+
+ ### Usage with Automod
+ Once you have configured the Phisherman plugin, you are ready to use it with automod. Currently, Phisherman is available as an option in the \`match_links\` plugin:
+
+ ~~~yml
+ automod:
+ config:
+ rules:
+ # Clean any scam links detected by Phisherman
+ filter_scam_links:
+ triggers:
+ - match_links:
+ phisherman:
+ include_suspected: true # It's recommended to keep this enabled to catch new scam domains quickly
+ include_verified: true
+ actions:
+ clean: true
+ ~~~
+ `),
+};
diff --git a/backend/src/plugins/Phisherman/types.ts b/backend/src/plugins/Phisherman/types.ts
new file mode 100644
index 00000000..56ed7aca
--- /dev/null
+++ b/backend/src/plugins/Phisherman/types.ts
@@ -0,0 +1,16 @@
+import * as t from "io-ts";
+import { BasePluginType } from "knub";
+import { tNullable } from "../../utils";
+
+export const ConfigSchema = t.type({
+ api_key: tNullable(t.string),
+});
+export type TConfigSchema = t.TypeOf<typeof ConfigSchema>;
+
+export interface PhishermanPluginType extends BasePluginType {
+ config: TConfigSchema;
+
+ state: {
+ validApiKey: string | null;
+ };
+}
diff --git a/backend/src/plugins/availablePlugins.ts b/backend/src/plugins/availablePlugins.ts
index 34535b7c..99b05f4c 100644
--- a/backend/src/plugins/availablePlugins.ts
+++ b/backend/src/plugins/availablePlugins.ts
@@ -34,6 +34,7 @@ import { UsernameSaverPlugin } from "./UsernameSaver/UsernameSaverPlugin";
import { UtilityPlugin } from "./Utility/UtilityPlugin";
import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin";
import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint";
+import { PhishermanPlugin } from "./Phisherman/PhishermanPlugin";
// prettier-ignore
export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
@@ -69,6 +70,7 @@ export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
TimeAndDatePlugin,
CountersPlugin,
ContextMenuPlugin,
+ PhishermanPlugin,
];
// prettier-ignore