feat: Phisherman integration

backend/bot.env.example

@@ -4,3 +4,4 @@ DB_USER=
 DB_PASSWORD=
 DB_DATABASE=
 PROFILING=false
+PHISHERMAN_API_KEY=

backend/package-lock.json

@@ -36,6 +36,7 @@
         "lodash.pick": "^4.4.0",
         "moment-timezone": "^0.5.21",
         "mysql": "^2.16.0",
+        "node-fetch": "^2.6.5",
         "parse-color": "^1.0.0",
         "passport": "^0.4.0",
         "passport-custom": "^1.0.5",
@@ -66,6 +67,7 @@
         "@types/lodash.at": "^4.6.3",
         "@types/moment-timezone": "^0.5.6",
         "@types/node": "^14.0.14",
+        "@types/node-fetch": "^2.5.12",
         "@types/passport": "^1.0.0",
         "@types/passport-oauth2": "^1.4.8",
         "@types/passport-strategy": "^0.2.35",
@@ -75,8 +77,7 @@
         "ava": "^3.10.0",
         "rimraf": "^2.6.2",
         "source-map-support": "^0.5.16",
-        "tsc-watch": "^4.0.0",
+        "tsc-watch": "^4.0.0"
-        "typescript": "^4.3.4"
       }
     },
     "../../Knub": {
@@ -424,6 +425,16 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
       "integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
     },
+    "node_modules/@types/node-fetch": {
+      "version": "2.5.12",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
+      "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
+      "dev": true,
+      "dependencies": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      }
+    },
     "node_modules/@types/normalize-package-data": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
@@ -2436,6 +2447,20 @@
         "node": ">=6"
       }
     },
+    "node_modules/form-data": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+      "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+      "dev": true,
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/forwarded": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
@@ -3513,9 +3538,12 @@
       "dev": true
     },
     "node_modules/node-fetch": {
-      "version": "2.6.1",
+      "version": "2.6.5",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
-      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==",
+      "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
       "engines": {
         "node": "4.x || >=6.0.0"
       }
@@ -5127,6 +5155,11 @@
         "node": ">=0.6"
       }
     },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+    },
     "node_modules/trim-off-newlines": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
@@ -5430,9 +5463,10 @@
       }
     },
     "node_modules/typescript": {
-      "version": "4.3.4",
+      "version": "4.4.4",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
-      "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==",
+      "integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
+      "peer": true,
       "bin": {
         "tsc": "bin/tsc",
         "tsserver": "bin/tsserver"
@@ -5647,6 +5681,11 @@
         "defaults": "^1.0.3"
       }
     },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+    },
     "node_modules/well-known-symbols": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
@@ -5656,6 +5695,15 @@
         "node": ">=6"
       }
     },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
     "node_modules/which": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",
@@ -6344,6 +6392,16 @@
       "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.14.tgz",
       "integrity": "sha512-syUgf67ZQpaJj01/tRTknkMNoBBLWJOBODF0Zm4NrXmiSuxjymFrxnTu1QVYRubhVkRcZLYZG8STTwJRdVm/WQ=="
     },
+    "@types/node-fetch": {
+      "version": "2.5.12",
+      "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz",
+      "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==",
+      "dev": true,
+      "requires": {
+        "@types/node": "*",
+        "form-data": "^3.0.0"
+      }
+    },
     "@types/normalize-package-data": {
       "version": "2.4.0",
       "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
@@ -7918,6 +7976,17 @@
         "locate-path": "^3.0.0"
       }
     },
+    "form-data": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+      "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
+      "dev": true,
+      "requires": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      }
+    },
     "forwarded": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
@@ -8747,9 +8816,12 @@
       "dev": true
     },
     "node-fetch": {
-      "version": "2.6.1",
+      "version": "2.6.5",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.5.tgz",
-      "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
+      "integrity": "sha512-mmlIVHJEu5rnIxgEgez6b9GgWXbkZj5YZ7fx+2r94a2E+Uirsp6HsPTPlomfdHtpt/B0cdKviwkoaM6pyvUOpQ==",
+      "requires": {
+        "whatwg-url": "^5.0.0"
+      }
     },
     "node-gyp-build": {
       "version": "4.2.3",
@@ -9992,6 +10064,11 @@
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz",
       "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw=="
     },
+    "tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o="
+    },
     "trim-off-newlines": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/trim-off-newlines/-/trim-off-newlines-1.0.1.tgz",
@@ -10220,9 +10297,10 @@
       }
     },
     "typescript": {
-      "version": "4.3.4",
+      "version": "4.4.4",
-      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz",
+      "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.4.4.tgz",
-      "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew=="
+      "integrity": "sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==",
+      "peer": true
     },
     "uid2": {
       "version": "0.0.3",
@@ -10382,12 +10460,26 @@
         "defaults": "^1.0.3"
       }
     },
+    "webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE="
+    },
     "well-known-symbols": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/well-known-symbols/-/well-known-symbols-2.0.0.tgz",
       "integrity": "sha512-ZMjC3ho+KXo0BfJb7JgtQ5IBuvnShdlACNkKkdsqBmYw3bPAaJfPeYUo6tLUaT5tG/Gkh7xkpBhKRQ9e7pyg9Q==",
       "dev": true
     },
+    "whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha1-lmRU6HZUYuN2RNNib2dCzotwll0=",
+      "requires": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
     "which": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz",

backend/package.json

@@ -51,6 +51,7 @@
     "lodash.pick": "^4.4.0",
     "moment-timezone": "^0.5.21",
     "mysql": "^2.16.0",
+    "node-fetch": "^2.6.5",
     "parse-color": "^1.0.0",
     "passport": "^0.4.0",
     "passport-custom": "^1.0.5",
@@ -81,6 +82,7 @@
     "@types/lodash.at": "^4.6.3",
     "@types/moment-timezone": "^0.5.6",
     "@types/node": "^14.0.14",
+    "@types/node-fetch": "^2.5.12",
     "@types/passport": "^1.0.0",
     "@types/passport-oauth2": "^1.4.8",
     "@types/passport-strategy": "^0.2.35",
@@ -90,8 +92,7 @@
     "ava": "^3.10.0",
     "rimraf": "^2.6.2",
     "source-map-support": "^0.5.16",
-    "tsc-watch": "^4.0.0",
+    "tsc-watch": "^4.0.0"
-    "typescript": "^4.3.4"
   },
   "ava": {
     "files": [

backend/src/data/Phisherman.ts

@@ -0,0 +1,229 @@
+import { getRepository, Repository } from "typeorm";
+import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry";
+import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman";
+import fetch, { Headers } from "node-fetch";
+import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
+import moment from "moment-timezone";
+import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
+import crypto from "crypto";
+
+const API_URL = "https://api.phisherman.gg";
+const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
+
+let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
+
+const pendingApiRequests: Map<string, Promise<unknown>> = new Map();
+const pendingDomainInfoChecks: Map<string, Promise<PhishermanDomainInfo | null>> = new Map();
+
+type MemoryCacheEntry = {
+  info: PhishermanDomainInfo | null;
+  expires: number;
+};
+const memoryCache: Map<string, MemoryCacheEntry> = new Map();
+
+setInterval(() => {
+  const now = Date.now();
+  for (const [key, entry] of memoryCache.entries()) {
+    if (entry.expires <= now) {
+      memoryCache.delete(key);
+    }
+  }
+}, 2 * MINUTES);
+
+const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES;
+const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES;
+const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS;
+
+const KEY_VALIDITY_LIFETIME = 24 * HOURS;
+
+let cacheRepository: Repository<PhishermanCacheEntry> | null = null;
+function getCacheRepository(): Repository<PhishermanCacheEntry> {
+  if (cacheRepository == null) {
+    cacheRepository = getRepository(PhishermanCacheEntry);
+  }
+  return cacheRepository;
+}
+
+let keyCacheRepository: Repository<PhishermanKeyCacheEntry> | null = null;
+function getKeyCacheRepository(): Repository<PhishermanKeyCacheEntry> {
+  if (keyCacheRepository == null) {
+    keyCacheRepository = getRepository(PhishermanKeyCacheEntry);
+  }
+  return keyCacheRepository;
+}
+
+class PhishermanApiError extends Error {
+  status: number;
+  constructor(status: number, message: string) {
+    super(message);
+    this.status = status;
+  }
+}
+
+export function hasPhishermanMasterAPIKey() {
+  return MASTER_API_KEY != null && MASTER_API_KEY !== "";
+}
+
+export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean {
+  return info.classification === "safe";
+}
+
+const leadingSlashRegex = /^\/+/g;
+function trimLeadingSlash(str: string): string {
+  return str.replace(leadingSlashRegex, "");
+}
+
+/**
+ * Make an arbitrary API call to the Phisherman API
+ */
+async function apiCall<T>(
+  method: "GET" | "POST",
+  resource: string,
+  payload?: Record<string, unknown> | null,
+): Promise<T> {
+  if (!hasPhishermanMasterAPIKey()) {
+    throw new Error("Phisherman master API key missing");
+  }
+
+  const url = `${API_URL}/${trimLeadingSlash(resource)}`;
+  const key = `${method} ${url}`;
+
+  if (pendingApiRequests.has(key)) {
+    return pendingApiRequests.get(key)! as Promise<T>;
+  }
+
+  const requestPromise = (async () => {
+    const response = await fetch(url, {
+      method,
+      headers: new Headers({
+        Accept: "application/json",
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${MASTER_API_KEY}`,
+      }),
+      body: payload ? JSON.stringify(payload) : undefined,
+    });
+    const data = await response.json().catch(() => null);
+    if (!response.ok || (data as any)?.success === false) {
+      throw new PhishermanApiError(response.status, (data as any)?.message ?? "");
+    }
+    return data;
+  })();
+  requestPromise.finally(() => {
+    pendingApiRequests.delete(key);
+  });
+  pendingApiRequests.set(key, requestPromise);
+  return requestPromise as Promise<T>;
+}
+
+type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo;
+async function fetchDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
+  // tslint:disable-next-line:no-console
+  console.log(`[PHISHERMAN] Requesting domain information: ${domain}`);
+  const result = await apiCall<Record<string, DomainInfoApiCallResult>>("GET", `/v2/domains/info/${domain}`);
+  const domainInfo = result[domain];
+  if (domainInfo.classification === "unknown") {
+    return null;
+  }
+  return domainInfo;
+}
+
+export async function getPhishermanDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
+  if (pendingDomainInfoChecks.has(domain)) {
+    return pendingDomainInfoChecks.get(domain)!;
+  }
+
+  const promise = (async () => {
+    if (memoryCache.has(domain)) {
+      return memoryCache.get(domain)!.info;
+    }
+
+    const dbCache = getCacheRepository();
+    const existingCachedEntry = await dbCache.findOne({ domain });
+    if (existingCachedEntry) {
+      return existingCachedEntry.data;
+    }
+
+    const freshData = await fetchDomainInfo(domain);
+    const expiryTime =
+      freshData === null
+        ? UNKNOWN_DOMAIN_CACHE_LIFETIME
+        : phishermanDomainIsSafe(freshData)
+        ? SAFE_DOMAIN_CACHE_LIFETIME
+        : DETECTED_DOMAIN_CACHE_LIFETIME;
+    memoryCache.set(domain, {
+      info: freshData,
+      expires: Date.now() + expiryTime,
+    });
+
+    if (freshData) {
+      // Database cache only stores safe/detected domains, not unknown ones
+      await dbCache.insert({
+        domain,
+        data: freshData,
+        expires_at: moment().add(expiryTime, "ms").format(DBDateFormat),
+      });
+    }
+
+    return freshData;
+  })();
+  promise.finally(() => {
+    pendingDomainInfoChecks.delete(domain);
+  });
+  pendingDomainInfoChecks.set(domain, promise);
+
+  return promise;
+}
+
+export async function phishermanApiKeyIsValid(apiKey: string): Promise<boolean> {
+  const keyCache = getKeyCacheRepository();
+  const hash = crypto.createHash("sha256").update(apiKey).digest("hex");
+  const entry = await keyCache.findOne({ hash });
+  if (entry) {
+    return entry.is_valid;
+  }
+
+  const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey });
+
+  await keyCache.insert({
+    hash,
+    is_valid: isValid,
+    expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat),
+  });
+
+  return isValid;
+}
+
+export function trackPhishermanCaughtDomain(apiKey: string, domain: string) {
+  if (!caughtDomainTrackingMap.has(apiKey)) {
+    caughtDomainTrackingMap.set(apiKey, new Map());
+  }
+  const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!;
+  if (!apiKeyMap.has(domain)) {
+    apiKeyMap.set(domain, []);
+  }
+  const timestamps = apiKeyMap.get(domain)!;
+  timestamps.push(Date.now());
+}
+
+export async function reportTrackedDomainsToPhisherman() {
+  const result = {};
+  for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) {
+    result[apiKey] = {};
+    for (const [domain, timestamps] of domains.entries()) {
+      result[apiKey][domain] = timestamps;
+    }
+  }
+
+  if (Object.keys(result).length > 0) {
+    await apiCall("POST", "/v2/phish/caught/bulk", result);
+    caughtDomainTrackingMap = new Map();
+  }
+}
+
+export async function deleteStalePhishermanCacheEntries() {
+  await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
+}
+
+export async function deleteStalePhishermanKeyCacheEntries() {
+  await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
+}

backend/src/data/entities/PhishermanCacheEntry.ts

@@ -0,0 +1,18 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+import { PhishermanDomainInfo } from "../types/phisherman";
+
+@Entity("phisherman_cache")
+export class PhishermanCacheEntry {
+  @Column()
+  @PrimaryColumn()
+  id: number;
+
+  @Column()
+  domain: string;
+
+  @Column("simple-json")
+  data: PhishermanDomainInfo;
+
+  @Column()
+  expires_at: string;
+}

backend/src/data/entities/PhishermanKeyCacheEntry.ts

@@ -0,0 +1,17 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+
+@Entity("phisherman_key_cache")
+export class PhishermanKeyCacheEntry {
+  @Column()
+  @PrimaryColumn()
+  id: number;
+
+  @Column()
+  hash: string;
+
+  @Column()
+  is_valid: boolean;
+
+  @Column()
+  expires_at: string;
+}

backend/src/data/loops/phishermanLoops.ts

@@ -0,0 +1,26 @@
+import { HOURS, MINUTES } from "../../utils";
+import {
+  deleteStalePhishermanCacheEntries,
+  deleteStalePhishermanKeyCacheEntries,
+  reportTrackedDomainsToPhisherman,
+} from "../Phisherman";
+
+const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES;
+const REPORT_LOOP_INTERVAL = 15 * MINUTES;
+
+export async function runPhishermanCacheCleanupLoop() {
+  console.log("[PHISHERMAN] Deleting stale cache entries");
+  await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err));
+
+  console.log("[PHISHERMAN] Deleting stale key cache entries");
+  await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err));
+
+  setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL);
+}
+
+export async function runPhishermanReportingLoop() {
+  console.log("[PHISHERMAN] Reporting tracked domains");
+  await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err));
+
+  setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL);
+}

backend/src/data/types/phisherman.ts

@@ -0,0 +1,32 @@
+export interface PhishermanUnknownDomain {
+  classification: "unknown";
+}
+
+export interface PhishermanDomainInfo {
+  status: string;
+  lastChecked: string;
+  verifiedPhish: boolean;
+  classification: "safe" | "malicious";
+  created: string;
+  firstSeen: string | null;
+  lastSeen: string | null;
+  targetedBrand: string;
+  phishCaught: number;
+  details: PhishermanDomainInfoDetails;
+}
+
+export interface PhishermanDomainInfoDetails {
+  phishTankId: string | null;
+  urlScanId: string;
+  websiteScreenshot: string;
+  ip_address: string;
+  asn: PhishermanDomainInfoAsn;
+  registry: string;
+  country: string;
+}
+
+export interface PhishermanDomainInfoAsn {
+  asn: string;
+  asn_name: string;
+  route: string;
+}

backend/src/index.ts

@@ -33,6 +33,8 @@ import { runSavedMessageCleanupLoop } from "./data/loops/savedMessageCleanupLoop
 import { performance } from "perf_hooks";
 import { setProfiler } from "./profiler";
 import { enableProfiling } from "./utils/easyProfiler";
+import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./data/loops/phishermanLoops";
+import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
 
 if (!process.env.KEY) {
   // tslint:disable-next-line:no-console
@@ -363,6 +365,13 @@ connect().then(async () => {
     runExpiredArchiveDeletionLoop();
     await sleep(10 * SECONDS);
     runSavedMessageCleanupLoop();
+
+    if (hasPhishermanMasterAPIKey()) {
+      await sleep(10 * SECONDS);
+      runPhishermanCacheCleanupLoop();
+      await sleep(10 * SECONDS);
+      runPhishermanReportingLoop();
+    }
   });
 
   setProfiler(bot.profiler);

backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts

@@ -0,0 +1,38 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreatePhishermanCacheTable1634563901575 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: "phisherman_cache",
+        columns: [
+          {
+            name: "id",
+            type: "int",
+            isPrimary: true,
+            isGenerated: true,
+            generationStrategy: "increment",
+          },
+          {
+            name: "domain",
+            type: "varchar",
+            length: "255",
+            isUnique: true,
+          },
+          {
+            name: "data",
+            type: "text",
+          },
+          {
+            name: "expires_at",
+            type: "datetime",
+          },
+        ],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropTable("phisherman_cache");
+  }
+}

backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts

@@ -0,0 +1,38 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreatePhishermanKeyCacheTable1635596150234 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: "phisherman_key_cache",
+        columns: [
+          {
+            name: "id",
+            type: "int",
+            isPrimary: true,
+            isGenerated: true,
+            generationStrategy: "increment",
+          },
+          {
+            name: "hash",
+            type: "varchar",
+            length: "255",
+            isUnique: true,
+          },
+          {
+            name: "is_valid",
+            type: "tinyint",
+          },
+          {
+            name: "expires_at",
+            type: "datetime",
+          },
+        ],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropTable("phisherman_key_cache");
+  }
+}

backend/src/plugins/Automod/AutomodPlugin.ts

@@ -30,6 +30,7 @@ import { clearOldRecentSpam } from "./functions/clearOldRecentSpam";
 import { pluginInfo } from "./info";
 import { availableTriggers } from "./triggers/availableTriggers";
 import { AutomodPluginType, ConfigSchema } from "./types";
+import { PhishermanPlugin } from "../Phisherman/PhishermanPlugin";
 
 const defaultOptions = {
   config: {
@@ -183,6 +184,7 @@ export const AutomodPlugin = zeppelinGuildPlugin<AutomodPluginType>()({
     ModActionsPlugin,
     MutesPlugin,
     CountersPlugin,
+    PhishermanPlugin,
   ],
 
   configSchema: ConfigSchema,

backend/src/plugins/Automod/triggers/matchLinks.ts

@@ -9,10 +9,13 @@ import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions
 import { automodTrigger } from "../helpers";
 import { mergeRegexes } from "../../../utils/mergeRegexes";
 import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex";
+import { PhishermanPlugin } from "../../Phisherman/PhishermanPlugin";
+import { phishermanDomainIsSafe } from "../../../data/Phisherman";
 
 interface MatchResultType {
   type: MatchableTextType;
   link: string;
+  details?: string;
 }
 
 const regexCache = new WeakMap<any, RegExp[]>();
@@ -28,6 +31,12 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
     exclude_words: tNullable(t.array(t.string)),
     include_regex: tNullable(t.array(TRegex)),
     exclude_regex: tNullable(t.array(TRegex)),
+    phisherman: tNullable(
+      t.type({
+        include_suspected: tNullable(t.boolean),
+        include_verified: tNullable(t.boolean),
+      }),
+    ),
     only_real_links: t.boolean,
     match_messages: t.boolean,
     match_embeds: t.boolean,
@@ -150,6 +159,25 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
             }
           }
         }
+
+        if (trigger.phisherman) {
+          const phishermanResult = await pluginData.getPlugin(PhishermanPlugin).getDomainInfo(normalizedHostname);
+          if (phishermanResult != null && !phishermanDomainIsSafe(phishermanResult)) {
+            if (
+              (trigger.phisherman.include_suspected && !phishermanResult.verifiedPhish) ||
+              (trigger.phisherman.include_verified && phishermanResult.verifiedPhish)
+            ) {
+              const suspectedVerified = phishermanResult.verifiedPhish ? "verified" : "suspected";
+              return {
+                extra: {
+                  type,
+                  link: link.input,
+                  details: `using Phisherman (${suspectedVerified})`,
+                },
+              };
+            }
+          }
+        }
       }
     }
 
@@ -158,6 +186,11 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
 
   renderMatchInformation({ pluginData, contexts, matchResult }) {
     const partialSummary = getTextMatchPartialSummary(pluginData, matchResult.extra.type, contexts[0]);
-    return `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\` in ${partialSummary}`;
+    let information = `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\``;
+    if (matchResult.extra.details) {
+      information += ` ${matchResult.extra.details}`;
+    }
+    information += ` in ${partialSummary}`;
+    return information;
   },
 });

backend/src/plugins/Phisherman/PhishermanPlugin.ts

@@ -0,0 +1,49 @@
+import { PluginOptions, typedGuildCommand } from "knub";
+import { GuildPingableRoles } from "../../data/GuildPingableRoles";
+import { zeppelinGuildPlugin } from "../ZeppelinPluginBlueprint";
+import { ConfigSchema, PhishermanPluginType } from "./types";
+import {
+  getPhishermanDomainInfo,
+  hasPhishermanMasterAPIKey,
+  phishermanApiKeyIsValid,
+  reportTrackedDomainsToPhisherman,
+} from "../../data/Phisherman";
+import { mapToPublicFn } from "../../pluginUtils";
+import { getDomainInfo } from "./functions/getDomainInfo";
+import { pluginInfo } from "./info";
+
+const defaultOptions: PluginOptions<PhishermanPluginType> = {
+  config: {
+    api_key: null,
+  },
+  overrides: [],
+};
+
+export const PhishermanPlugin = zeppelinGuildPlugin<PhishermanPluginType>()({
+  name: "phisherman",
+  showInDocs: true,
+  info: pluginInfo,
+
+  configSchema: ConfigSchema,
+  defaultOptions,
+
+  // prettier-ignore
+  public: {
+    getDomainInfo: mapToPublicFn(getDomainInfo),
+  },
+
+  async beforeLoad(pluginData) {
+    pluginData.state.validApiKey = null;
+
+    if (!hasPhishermanMasterAPIKey()) {
+      // tslint:disable-next-line:no-console
+      console.warn("Could not load Phisherman plugin: master API key is missing");
+      return;
+    }
+
+    const apiKey = pluginData.config.get().api_key;
+    if (apiKey && (await phishermanApiKeyIsValid(apiKey).catch(() => false))) {
+      pluginData.state.validApiKey = apiKey;
+    }
+  },
+});

backend/src/plugins/Phisherman/functions/getDomainInfo.ts

@@ -0,0 +1,20 @@
+import { GuildPluginData } from "knub";
+import { PhishermanPluginType } from "../types";
+import { PhishermanDomainInfo } from "../../../data/types/phisherman";
+import { getPhishermanDomainInfo, phishermanDomainIsSafe, trackPhishermanCaughtDomain } from "../../../data/Phisherman";
+
+export async function getDomainInfo(
+  pluginData: GuildPluginData<PhishermanPluginType>,
+  domain: string,
+): Promise<PhishermanDomainInfo | null> {
+  if (!pluginData.state.validApiKey) {
+    return null;
+  }
+
+  const info = await getPhishermanDomainInfo(domain).catch(() => null);
+  if (info != null && !phishermanDomainIsSafe(info)) {
+    trackPhishermanCaughtDomain(pluginData.state.validApiKey, domain);
+  }
+
+  return info;
+}

backend/src/plugins/Phisherman/info.ts

@@ -0,0 +1,41 @@
+import { trimPluginDescription } from "../../utils";
+import { ZeppelinGuildPluginBlueprint } from "../ZeppelinPluginBlueprint";
+
+export const pluginInfo: ZeppelinGuildPluginBlueprint["info"] = {
+  prettyName: "Phisherman",
+  description: trimPluginDescription(`
+    Match scam/phishing links using the Phisherman API. See https://phisherman.gg/ for more details!
+  `),
+  configurationGuide: trimPluginDescription(`
+    ### Getting started
+    To get started, request an API key for Phisherman following the instructions at https://docs.phisherman.gg/#/api/getting-started?id=requesting-api-access.
+    Then, add the api key to the plugin's config:
+    
+    ~~~yml
+    phisherman:
+      config:
+        api_key: "your key here"
+    ~~~
+    
+    ### Note
+    When using Phisherman features in Zeppelin, Zeppelin reports statistics about checked links back to Phisherman. This only includes the domain (e.g. zeppelin.gg), not the full link.
+    
+    ### Usage with Automod
+    Once you have configured the Phisherman plugin, you are ready to use it with automod. Currently, Phisherman is available as an option in the \`match_links\` plugin:
+    
+    ~~~yml
+    automod:
+      config:
+        rules:
+          # Clean any scam links detected by Phisherman
+          filter_scam_links:
+            triggers:
+              - match_links:
+                  phisherman:
+                    include_suspected: true # It's recommended to keep this enabled to catch new scam domains quickly
+                    include_verified: true
+            actions:
+              clean: true
+    ~~~
+  `),
+};

backend/src/plugins/Phisherman/types.ts

@@ -0,0 +1,16 @@
+import * as t from "io-ts";
+import { BasePluginType } from "knub";
+import { tNullable } from "../../utils";
+
+export const ConfigSchema = t.type({
+  api_key: tNullable(t.string),
+});
+export type TConfigSchema = t.TypeOf<typeof ConfigSchema>;
+
+export interface PhishermanPluginType extends BasePluginType {
+  config: TConfigSchema;
+
+  state: {
+    validApiKey: string | null;
+  };
+}

backend/src/plugins/availablePlugins.ts

@@ -34,6 +34,7 @@ import { UsernameSaverPlugin } from "./UsernameSaver/UsernameSaverPlugin";
 import { UtilityPlugin } from "./Utility/UtilityPlugin";
 import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin";
 import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint";
+import { PhishermanPlugin } from "./Phisherman/PhishermanPlugin";
 
 // prettier-ignore
 export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
@@ -69,6 +70,7 @@ export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
   TimeAndDatePlugin,
   CountersPlugin,
   ContextMenuPlugin,
+  PhishermanPlugin,
 ];
 
 // prettier-ignore