From 13c94a81cc5f91c05f2fcdc692d31b8cde192244 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 31 Oct 2021 17:17:31 +0200 Subject: [PATCH] feat: Phisherman integration --- backend/bot.env.example | 1 + backend/package-lock.json | 120 +++++++-- backend/package.json | 5 +- backend/src/data/Phisherman.ts | 229 ++++++++++++++++++ .../src/data/entities/PhishermanCacheEntry.ts | 18 ++ .../data/entities/PhishermanKeyCacheEntry.ts | 17 ++ backend/src/data/loops/phishermanLoops.ts | 26 ++ backend/src/data/types/phisherman.ts | 32 +++ backend/src/index.ts | 9 + ...634563901575-CreatePhishermanCacheTable.ts | 38 +++ ...596150234-CreatePhishermanKeyCacheTable.ts | 38 +++ backend/src/plugins/Automod/AutomodPlugin.ts | 2 + .../plugins/Automod/triggers/matchLinks.ts | 35 ++- .../plugins/Phisherman/PhishermanPlugin.ts | 49 ++++ .../Phisherman/functions/getDomainInfo.ts | 20 ++ backend/src/plugins/Phisherman/info.ts | 41 ++++ backend/src/plugins/Phisherman/types.ts | 16 ++ backend/src/plugins/availablePlugins.ts | 2 + 18 files changed, 681 insertions(+), 17 deletions(-) create mode 100644 backend/src/data/Phisherman.ts create mode 100644 backend/src/data/entities/PhishermanCacheEntry.ts create mode 100644 backend/src/data/entities/PhishermanKeyCacheEntry.ts create mode 100644 backend/src/data/loops/phishermanLoops.ts create mode 100644 backend/src/data/types/phisherman.ts create mode 100644 backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts create mode 100644 backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts create mode 100644 backend/src/plugins/Phisherman/PhishermanPlugin.ts create mode 100644 backend/src/plugins/Phisherman/functions/getDomainInfo.ts create mode 100644 backend/src/plugins/Phisherman/info.ts create mode 100644 backend/src/plugins/Phisherman/types.ts diff --git a/backend/bot.env.example b/backend/bot.env.example index 7fdd7e5c..b701f4ed 100644 --- a/backend/bot.env.example +++ b/backend/bot.env.example @@ -4,3 +4,4 @@ DB_USER= DB_PASSWORD= DB_DATABASE= PROFILING=false +PHISHERMAN_API_KEY= diff --git a/backend/package-lock.json b/backend/package-lock.json index cdc6456a..a5e5cf8b 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -36,6 +36,7 @@ "lodash.pick": "^4.4.0", "moment-timezone": "^0.5.21", "mysql": "^2.16.0", + "node-fetch": "^2.6.5", "parse-color": "^1.0.0", "passport": "^0.4.0", "passport-custom": "^1.0.5", @@ -66,6 +67,7 @@ "@types/lodash.at": "^4.6.3", "@types/moment-timezone": "^0.5.6", "@types/node": "^14.0.14", + "@types/node-fetch": "^2.5.12", "@types/passport": "^1.0.0", "@types/passport-oauth2": "^1.4.8", "@types/passport-strategy": "^0.2.35", @@ -75,8 +77,7 @@ "ava": "^3.10.0", "rimraf": "^2.6.2", "source-map-support": "^0.5.16", - "tsc-watch": "^4.0.0", - "typescript": "^4.3.4" + "tsc-watch": "^4.0.0" } }, "../../Knub": { @@ -424,6 +425,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz", "integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ==" }, + "node_modules/@types/node-fetch": { + "version": "2.5.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz", + "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "node_modules/@types/normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -2436,6 +2447,20 @@ "node": ">=6" } }, + "node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", @@ -3513,9 +3538,12 @@ "dev": true }, "node_modules/node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz", + "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==", + "dependencies": { + "whatwg-url": "^5.0.0" + }, "engines": { "node": "4.x || >=6.0.0" } @@ -5127,6 +5155,11 @@ "node": ">=0.6" } }, + "node_modules/tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, "node_modules/trim-off-newlines": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz", @@ -5430,9 +5463,10 @@ } }, "node_modules/typescript": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz", - "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==", + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz", + "integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -5647,6 +5681,11 @@ "defaults": "^1.0.3" } }, + "node_modules/webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, "node_modules/well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", @@ -5656,6 +5695,15 @@ "node": ">=6" } }, + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "dependencies": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "node_modules/which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", @@ -6344,6 +6392,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz", "integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ==" }, + "@types/node-fetch": { + "version": "2.5.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz", + "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "@types/normalize-package-data": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz", @@ -7918,6 +7976,17 @@ "locate-path": "^3.0.0" } }, + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "forwarded": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", @@ -8747,9 +8816,12 @@ "dev": true }, "node-fetch": { - "version": "2.6.1", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", - "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + "version": "2.6.5", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz", + "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==", + "requires": { + "whatwg-url": "^5.0.0" + } }, "node-gyp-build": { "version": "4.2.3", @@ -9992,6 +10064,11 @@ "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" }, + "tr46": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", + "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" + }, "trim-off-newlines": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz", @@ -10220,9 +10297,10 @@ } }, "typescript": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz", - "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==" + "version": "4.4.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz", + "integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==", + "peer": true }, "uid2": { "version": "0.0.3", @@ -10382,12 +10460,26 @@ "defaults": "^1.0.3" } }, + "webidl-conversions": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", + "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=" + }, "well-known-symbols": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz", "integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==", "dev": true }, + "whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=", + "requires": { + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, "which": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", diff --git a/backend/package.json b/backend/package.json index a9870eec..1a977808 100644 --- a/backend/package.json +++ b/backend/package.json @@ -51,6 +51,7 @@ "lodash.pick": "^4.4.0", "moment-timezone": "^0.5.21", "mysql": "^2.16.0", + "node-fetch": "^2.6.5", "parse-color": "^1.0.0", "passport": "^0.4.0", "passport-custom": "^1.0.5", @@ -81,6 +82,7 @@ "@types/lodash.at": "^4.6.3", "@types/moment-timezone": "^0.5.6", "@types/node": "^14.0.14", + "@types/node-fetch": "^2.5.12", "@types/passport": "^1.0.0", "@types/passport-oauth2": "^1.4.8", "@types/passport-strategy": "^0.2.35", @@ -90,8 +92,7 @@ "ava": "^3.10.0", "rimraf": "^2.6.2", "source-map-support": "^0.5.16", - "tsc-watch": "^4.0.0", - "typescript": "^4.3.4" + "tsc-watch": "^4.0.0" }, "ava": { "files": [ diff --git a/backend/src/data/Phisherman.ts b/backend/src/data/Phisherman.ts new file mode 100644 index 00000000..25fb59ce --- /dev/null +++ b/backend/src/data/Phisherman.ts @@ -0,0 +1,229 @@ +import { getRepository, Repository } from "typeorm"; +import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry"; +import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman"; +import fetch, { Headers } from "node-fetch"; +import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils"; +import moment from "moment-timezone"; +import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry"; +import crypto from "crypto"; + +const API_URL = "https://api.phisherman.gg"; +const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY; + +let caughtDomainTrackingMap: Map> = new Map(); + +const pendingApiRequests: Map> = new Map(); +const pendingDomainInfoChecks: Map> = new Map(); + +type MemoryCacheEntry = { + info: PhishermanDomainInfo | null; + expires: number; +}; +const memoryCache: Map = new Map(); + +setInterval(() => { + const now = Date.now(); + for (const [key, entry] of memoryCache.entries()) { + if (entry.expires <= now) { + memoryCache.delete(key); + } + } +}, 2 * MINUTES); + +const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES; +const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES; +const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS; + +const KEY_VALIDITY_LIFETIME = 24 * HOURS; + +let cacheRepository: Repository | null = null; +function getCacheRepository(): Repository { + if (cacheRepository == null) { + cacheRepository = getRepository(PhishermanCacheEntry); + } + return cacheRepository; +} + +let keyCacheRepository: Repository | null = null; +function getKeyCacheRepository(): Repository { + if (keyCacheRepository == null) { + keyCacheRepository = getRepository(PhishermanKeyCacheEntry); + } + return keyCacheRepository; +} + +class PhishermanApiError extends Error { + status: number; + constructor(status: number, message: string) { + super(message); + this.status = status; + } +} + +export function hasPhishermanMasterAPIKey() { + return MASTER_API_KEY != null && MASTER_API_KEY !== ""; +} + +export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean { + return info.classification === "safe"; +} + +const leadingSlashRegex = /^\/+/g; +function trimLeadingSlash(str: string): string { + return str.replace(leadingSlashRegex, ""); +} + +/** + * Make an arbitrary API call to the Phisherman API + */ +async function apiCall( + method: "GET" | "POST", + resource: string, + payload?: Record | null, +): Promise { + if (!hasPhishermanMasterAPIKey()) { + throw new Error("Phisherman master API key missing"); + } + + const url = `${API_URL}/${trimLeadingSlash(resource)}`; + const key = `${method} ${url}`; + + if (pendingApiRequests.has(key)) { + return pendingApiRequests.get(key)! as Promise; + } + + const requestPromise = (async () => { + const response = await fetch(url, { + method, + headers: new Headers({ + Accept: "application/json", + "Content-Type": "application/json", + Authorization: `Bearer ${MASTER_API_KEY}`, + }), + body: payload ? JSON.stringify(payload) : undefined, + }); + const data = await response.json().catch(() => null); + if (!response.ok || (data as any)?.success === false) { + throw new PhishermanApiError(response.status, (data as any)?.message ?? ""); + } + return data; + })(); + requestPromise.finally(() => { + pendingApiRequests.delete(key); + }); + pendingApiRequests.set(key, requestPromise); + return requestPromise as Promise; +} + +type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo; +async function fetchDomainInfo(domain: string): Promise { + // tslint:disable-next-line:no-console + console.log(`[PHISHERMAN] Requesting domain information: ${domain}`); + const result = await apiCall>("GET", `/v2/domains/info/${domain}`); + const domainInfo = result[domain]; + if (domainInfo.classification === "unknown") { + return null; + } + return domainInfo; +} + +export async function getPhishermanDomainInfo(domain: string): Promise { + if (pendingDomainInfoChecks.has(domain)) { + return pendingDomainInfoChecks.get(domain)!; + } + + const promise = (async () => { + if (memoryCache.has(domain)) { + return memoryCache.get(domain)!.info; + } + + const dbCache = getCacheRepository(); + const existingCachedEntry = await dbCache.findOne({ domain }); + if (existingCachedEntry) { + return existingCachedEntry.data; + } + + const freshData = await fetchDomainInfo(domain); + const expiryTime = + freshData === null + ? UNKNOWN_DOMAIN_CACHE_LIFETIME + : phishermanDomainIsSafe(freshData) + ? SAFE_DOMAIN_CACHE_LIFETIME + : DETECTED_DOMAIN_CACHE_LIFETIME; + memoryCache.set(domain, { + info: freshData, + expires: Date.now() + expiryTime, + }); + + if (freshData) { + // Database cache only stores safe/detected domains, not unknown ones + await dbCache.insert({ + domain, + data: freshData, + expires_at: moment().add(expiryTime, "ms").format(DBDateFormat), + }); + } + + return freshData; + })(); + promise.finally(() => { + pendingDomainInfoChecks.delete(domain); + }); + pendingDomainInfoChecks.set(domain, promise); + + return promise; +} + +export async function phishermanApiKeyIsValid(apiKey: string): Promise { + const keyCache = getKeyCacheRepository(); + const hash = crypto.createHash("sha256").update(apiKey).digest("hex"); + const entry = await keyCache.findOne({ hash }); + if (entry) { + return entry.is_valid; + } + + const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey }); + + await keyCache.insert({ + hash, + is_valid: isValid, + expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat), + }); + + return isValid; +} + +export function trackPhishermanCaughtDomain(apiKey: string, domain: string) { + if (!caughtDomainTrackingMap.has(apiKey)) { + caughtDomainTrackingMap.set(apiKey, new Map()); + } + const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!; + if (!apiKeyMap.has(domain)) { + apiKeyMap.set(domain, []); + } + const timestamps = apiKeyMap.get(domain)!; + timestamps.push(Date.now()); +} + +export async function reportTrackedDomainsToPhisherman() { + const result = {}; + for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) { + result[apiKey] = {}; + for (const [domain, timestamps] of domains.entries()) { + result[apiKey][domain] = timestamps; + } + } + + if (Object.keys(result).length > 0) { + await apiCall("POST", "/v2/phish/caught/bulk", result); + caughtDomainTrackingMap = new Map(); + } +} + +export async function deleteStalePhishermanCacheEntries() { + await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute(); +} + +export async function deleteStalePhishermanKeyCacheEntries() { + await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute(); +} diff --git a/backend/src/data/entities/PhishermanCacheEntry.ts b/backend/src/data/entities/PhishermanCacheEntry.ts new file mode 100644 index 00000000..cad135bf --- /dev/null +++ b/backend/src/data/entities/PhishermanCacheEntry.ts @@ -0,0 +1,18 @@ +import { Column, Entity, PrimaryColumn } from "typeorm"; +import { PhishermanDomainInfo } from "../types/phisherman"; + +@Entity("phisherman_cache") +export class PhishermanCacheEntry { + @Column() + @PrimaryColumn() + id: number; + + @Column() + domain: string; + + @Column("simple-json") + data: PhishermanDomainInfo; + + @Column() + expires_at: string; +} diff --git a/backend/src/data/entities/PhishermanKeyCacheEntry.ts b/backend/src/data/entities/PhishermanKeyCacheEntry.ts new file mode 100644 index 00000000..c4286d1c --- /dev/null +++ b/backend/src/data/entities/PhishermanKeyCacheEntry.ts @@ -0,0 +1,17 @@ +import { Column, Entity, PrimaryColumn } from "typeorm"; + +@Entity("phisherman_key_cache") +export class PhishermanKeyCacheEntry { + @Column() + @PrimaryColumn() + id: number; + + @Column() + hash: string; + + @Column() + is_valid: boolean; + + @Column() + expires_at: string; +} diff --git a/backend/src/data/loops/phishermanLoops.ts b/backend/src/data/loops/phishermanLoops.ts new file mode 100644 index 00000000..507f2166 --- /dev/null +++ b/backend/src/data/loops/phishermanLoops.ts @@ -0,0 +1,26 @@ +import { HOURS, MINUTES } from "../../utils"; +import { + deleteStalePhishermanCacheEntries, + deleteStalePhishermanKeyCacheEntries, + reportTrackedDomainsToPhisherman, +} from "../Phisherman"; + +const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES; +const REPORT_LOOP_INTERVAL = 15 * MINUTES; + +export async function runPhishermanCacheCleanupLoop() { + console.log("[PHISHERMAN] Deleting stale cache entries"); + await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err)); + + console.log("[PHISHERMAN] Deleting stale key cache entries"); + await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err)); + + setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL); +} + +export async function runPhishermanReportingLoop() { + console.log("[PHISHERMAN] Reporting tracked domains"); + await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err)); + + setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL); +} diff --git a/backend/src/data/types/phisherman.ts b/backend/src/data/types/phisherman.ts new file mode 100644 index 00000000..418fd43a --- /dev/null +++ b/backend/src/data/types/phisherman.ts @@ -0,0 +1,32 @@ +export interface PhishermanUnknownDomain { + classification: "unknown"; +} + +export interface PhishermanDomainInfo { + status: string; + lastChecked: string; + verifiedPhish: boolean; + classification: "safe" | "malicious"; + created: string; + firstSeen: string | null; + lastSeen: string | null; + targetedBrand: string; + phishCaught: number; + details: PhishermanDomainInfoDetails; +} + +export interface PhishermanDomainInfoDetails { + phishTankId: string | null; + urlScanId: string; + websiteScreenshot: string; + ip_address: string; + asn: PhishermanDomainInfoAsn; + registry: string; + country: string; +} + +export interface PhishermanDomainInfoAsn { + asn: string; + asn_name: string; + route: string; +} diff --git a/backend/src/index.ts b/backend/src/index.ts index bbb4770b..587a515f 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -33,6 +33,8 @@ import { runSavedMessageCleanupLoop } from "./data/loops/savedMessageCleanupLoop import { performance } from "perf_hooks"; import { setProfiler } from "./profiler"; import { enableProfiling } from "./utils/easyProfiler"; +import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./data/loops/phishermanLoops"; +import { hasPhishermanMasterAPIKey } from "./data/Phisherman"; if (!process.env.KEY) { // tslint:disable-next-line:no-console @@ -363,6 +365,13 @@ connect().then(async () => { runExpiredArchiveDeletionLoop(); await sleep(10 * SECONDS); runSavedMessageCleanupLoop(); + + if (hasPhishermanMasterAPIKey()) { + await sleep(10 * SECONDS); + runPhishermanCacheCleanupLoop(); + await sleep(10 * SECONDS); + runPhishermanReportingLoop(); + } }); setProfiler(bot.profiler); diff --git a/backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts b/backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts new file mode 100644 index 00000000..a0f5543e --- /dev/null +++ b/backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts @@ -0,0 +1,38 @@ +import { MigrationInterface, QueryRunner, Table } from "typeorm"; + +export class CreatePhishermanCacheTable1634563901575 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable( + new Table({ + name: "phisherman_cache", + columns: [ + { + name: "id", + type: "int", + isPrimary: true, + isGenerated: true, + generationStrategy: "increment", + }, + { + name: "domain", + type: "varchar", + length: "255", + isUnique: true, + }, + { + name: "data", + type: "text", + }, + { + name: "expires_at", + type: "datetime", + }, + ], + }), + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable("phisherman_cache"); + } +} diff --git a/backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts b/backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts new file mode 100644 index 00000000..2e9bb7c7 --- /dev/null +++ b/backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts @@ -0,0 +1,38 @@ +import { MigrationInterface, QueryRunner, Table } from "typeorm"; + +export class CreatePhishermanKeyCacheTable1635596150234 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable( + new Table({ + name: "phisherman_key_cache", + columns: [ + { + name: "id", + type: "int", + isPrimary: true, + isGenerated: true, + generationStrategy: "increment", + }, + { + name: "hash", + type: "varchar", + length: "255", + isUnique: true, + }, + { + name: "is_valid", + type: "tinyint", + }, + { + name: "expires_at", + type: "datetime", + }, + ], + }), + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable("phisherman_key_cache"); + } +} diff --git a/backend/src/plugins/Automod/AutomodPlugin.ts b/backend/src/plugins/Automod/AutomodPlugin.ts index 602db445..16339309 100644 --- a/backend/src/plugins/Automod/AutomodPlugin.ts +++ b/backend/src/plugins/Automod/AutomodPlugin.ts @@ -30,6 +30,7 @@ import { clearOldRecentSpam } from "./functions/clearOldRecentSpam"; import { pluginInfo } from "./info"; import { availableTriggers } from "./triggers/availableTriggers"; import { AutomodPluginType, ConfigSchema } from "./types"; +import { PhishermanPlugin } from "../Phisherman/PhishermanPlugin"; const defaultOptions = { config: { @@ -183,6 +184,7 @@ export const AutomodPlugin = zeppelinGuildPlugin()({ ModActionsPlugin, MutesPlugin, CountersPlugin, + PhishermanPlugin, ], configSchema: ConfigSchema, diff --git a/backend/src/plugins/Automod/triggers/matchLinks.ts b/backend/src/plugins/Automod/triggers/matchLinks.ts index 31aa8006..248a19f3 100644 --- a/backend/src/plugins/Automod/triggers/matchLinks.ts +++ b/backend/src/plugins/Automod/triggers/matchLinks.ts @@ -9,10 +9,13 @@ import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions import { automodTrigger } from "../helpers"; import { mergeRegexes } from "../../../utils/mergeRegexes"; import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex"; +import { PhishermanPlugin } from "../../Phisherman/PhishermanPlugin"; +import { phishermanDomainIsSafe } from "../../../data/Phisherman"; interface MatchResultType { type: MatchableTextType; link: string; + details?: string; } const regexCache = new WeakMap(); @@ -28,6 +31,12 @@ export const MatchLinksTrigger = automodTrigger()({ exclude_words: tNullable(t.array(t.string)), include_regex: tNullable(t.array(TRegex)), exclude_regex: tNullable(t.array(TRegex)), + phisherman: tNullable( + t.type({ + include_suspected: tNullable(t.boolean), + include_verified: tNullable(t.boolean), + }), + ), only_real_links: t.boolean, match_messages: t.boolean, match_embeds: t.boolean, @@ -150,6 +159,25 @@ export const MatchLinksTrigger = automodTrigger()({ } } } + + if (trigger.phisherman) { + const phishermanResult = await pluginData.getPlugin(PhishermanPlugin).getDomainInfo(normalizedHostname); + if (phishermanResult != null && !phishermanDomainIsSafe(phishermanResult)) { + if ( + (trigger.phisherman.include_suspected && !phishermanResult.verifiedPhish) || + (trigger.phisherman.include_verified && phishermanResult.verifiedPhish) + ) { + const suspectedVerified = phishermanResult.verifiedPhish ? "verified" : "suspected"; + return { + extra: { + type, + link: link.input, + details: `using Phisherman (${suspectedVerified})`, + }, + }; + } + } + } } } @@ -158,6 +186,11 @@ export const MatchLinksTrigger = automodTrigger()({ renderMatchInformation({ pluginData, contexts, matchResult }) { const partialSummary = getTextMatchPartialSummary(pluginData, matchResult.extra.type, contexts[0]); - return `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\` in ${partialSummary}`; + let information = `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\``; + if (matchResult.extra.details) { + information += ` ${matchResult.extra.details}`; + } + information += ` in ${partialSummary}`; + return information; }, }); diff --git a/backend/src/plugins/Phisherman/PhishermanPlugin.ts b/backend/src/plugins/Phisherman/PhishermanPlugin.ts new file mode 100644 index 00000000..50a6c321 --- /dev/null +++ b/backend/src/plugins/Phisherman/PhishermanPlugin.ts @@ -0,0 +1,49 @@ +import { PluginOptions, typedGuildCommand } from "knub"; +import { GuildPingableRoles } from "../../data/GuildPingableRoles"; +import { zeppelinGuildPlugin } from "../ZeppelinPluginBlueprint"; +import { ConfigSchema, PhishermanPluginType } from "./types"; +import { + getPhishermanDomainInfo, + hasPhishermanMasterAPIKey, + phishermanApiKeyIsValid, + reportTrackedDomainsToPhisherman, +} from "../../data/Phisherman"; +import { mapToPublicFn } from "../../pluginUtils"; +import { getDomainInfo } from "./functions/getDomainInfo"; +import { pluginInfo } from "./info"; + +const defaultOptions: PluginOptions = { + config: { + api_key: null, + }, + overrides: [], +}; + +export const PhishermanPlugin = zeppelinGuildPlugin()({ + name: "phisherman", + showInDocs: true, + info: pluginInfo, + + configSchema: ConfigSchema, + defaultOptions, + + // prettier-ignore + public: { + getDomainInfo: mapToPublicFn(getDomainInfo), + }, + + async beforeLoad(pluginData) { + pluginData.state.validApiKey = null; + + if (!hasPhishermanMasterAPIKey()) { + // tslint:disable-next-line:no-console + console.warn("Could not load Phisherman plugin: master API key is missing"); + return; + } + + const apiKey = pluginData.config.get().api_key; + if (apiKey && (await phishermanApiKeyIsValid(apiKey).catch(() => false))) { + pluginData.state.validApiKey = apiKey; + } + }, +}); diff --git a/backend/src/plugins/Phisherman/functions/getDomainInfo.ts b/backend/src/plugins/Phisherman/functions/getDomainInfo.ts new file mode 100644 index 00000000..d5b9cbf3 --- /dev/null +++ b/backend/src/plugins/Phisherman/functions/getDomainInfo.ts @@ -0,0 +1,20 @@ +import { GuildPluginData } from "knub"; +import { PhishermanPluginType } from "../types"; +import { PhishermanDomainInfo } from "../../../data/types/phisherman"; +import { getPhishermanDomainInfo, phishermanDomainIsSafe, trackPhishermanCaughtDomain } from "../../../data/Phisherman"; + +export async function getDomainInfo( + pluginData: GuildPluginData, + domain: string, +): Promise { + if (!pluginData.state.validApiKey) { + return null; + } + + const info = await getPhishermanDomainInfo(domain).catch(() => null); + if (info != null && !phishermanDomainIsSafe(info)) { + trackPhishermanCaughtDomain(pluginData.state.validApiKey, domain); + } + + return info; +} diff --git a/backend/src/plugins/Phisherman/info.ts b/backend/src/plugins/Phisherman/info.ts new file mode 100644 index 00000000..5c9a1818 --- /dev/null +++ b/backend/src/plugins/Phisherman/info.ts @@ -0,0 +1,41 @@ +import { trimPluginDescription } from "../../utils"; +import { ZeppelinGuildPluginBlueprint } from "../ZeppelinPluginBlueprint"; + +export const pluginInfo: ZeppelinGuildPluginBlueprint["info"] = { + prettyName: "Phisherman", + description: trimPluginDescription(` + Match scam/phishing links using the Phisherman API. See https://phisherman.gg/ for more details! + `), + configurationGuide: trimPluginDescription(` + ### Getting started + To get started, request an API key for Phisherman following the instructions at https://docs.phisherman.gg/#/api/getting-started?id=requesting-api-access. + Then, add the api key to the plugin's config: + + ~~~yml + phisherman: + config: + api_key: "your key here" + ~~~ + + ### Note + When using Phisherman features in Zeppelin, Zeppelin reports statistics about checked links back to Phisherman. This only includes the domain (e.g. zeppelin.gg), not the full link. + + ### Usage with Automod + Once you have configured the Phisherman plugin, you are ready to use it with automod. Currently, Phisherman is available as an option in the \`match_links\` plugin: + + ~~~yml + automod: + config: + rules: + # Clean any scam links detected by Phisherman + filter_scam_links: + triggers: + - match_links: + phisherman: + include_suspected: true # It's recommended to keep this enabled to catch new scam domains quickly + include_verified: true + actions: + clean: true + ~~~ + `), +}; diff --git a/backend/src/plugins/Phisherman/types.ts b/backend/src/plugins/Phisherman/types.ts new file mode 100644 index 00000000..56ed7aca --- /dev/null +++ b/backend/src/plugins/Phisherman/types.ts @@ -0,0 +1,16 @@ +import * as t from "io-ts"; +import { BasePluginType } from "knub"; +import { tNullable } from "../../utils"; + +export const ConfigSchema = t.type({ + api_key: tNullable(t.string), +}); +export type TConfigSchema = t.TypeOf; + +export interface PhishermanPluginType extends BasePluginType { + config: TConfigSchema; + + state: { + validApiKey: string | null; + }; +} diff --git a/backend/src/plugins/availablePlugins.ts b/backend/src/plugins/availablePlugins.ts index 34535b7c..99b05f4c 100644 --- a/backend/src/plugins/availablePlugins.ts +++ b/backend/src/plugins/availablePlugins.ts @@ -34,6 +34,7 @@ import { UsernameSaverPlugin } from "./UsernameSaver/UsernameSaverPlugin"; import { UtilityPlugin } from "./Utility/UtilityPlugin"; import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin"; import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint"; +import { PhishermanPlugin } from "./Phisherman/PhishermanPlugin"; // prettier-ignore export const guildPlugins: Array> = [ @@ -69,6 +70,7 @@ export const guildPlugins: Array> = [ TimeAndDatePlugin, CountersPlugin, ContextMenuPlugin, + PhishermanPlugin, ]; // prettier-ignore