feat: Phisherman integration

backend/src/data/Phisherman.ts

@@ -0,0 +1,229 @@
+import { getRepository, Repository } from "typeorm";
+import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry";
+import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman";
+import fetch, { Headers } from "node-fetch";
+import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
+import moment from "moment-timezone";
+import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
+import crypto from "crypto";
+
+const API_URL = "https://api.phisherman.gg";
+const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
+
+let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
+
+const pendingApiRequests: Map<string, Promise<unknown>> = new Map();
+const pendingDomainInfoChecks: Map<string, Promise<PhishermanDomainInfo | null>> = new Map();
+
+type MemoryCacheEntry = {
+  info: PhishermanDomainInfo | null;
+  expires: number;
+};
+const memoryCache: Map<string, MemoryCacheEntry> = new Map();
+
+setInterval(() => {
+  const now = Date.now();
+  for (const [key, entry] of memoryCache.entries()) {
+    if (entry.expires <= now) {
+      memoryCache.delete(key);
+    }
+  }
+}, 2 * MINUTES);
+
+const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES;
+const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES;
+const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS;
+
+const KEY_VALIDITY_LIFETIME = 24 * HOURS;
+
+let cacheRepository: Repository<PhishermanCacheEntry> | null = null;
+function getCacheRepository(): Repository<PhishermanCacheEntry> {
+  if (cacheRepository == null) {
+    cacheRepository = getRepository(PhishermanCacheEntry);
+  }
+  return cacheRepository;
+}
+
+let keyCacheRepository: Repository<PhishermanKeyCacheEntry> | null = null;
+function getKeyCacheRepository(): Repository<PhishermanKeyCacheEntry> {
+  if (keyCacheRepository == null) {
+    keyCacheRepository = getRepository(PhishermanKeyCacheEntry);
+  }
+  return keyCacheRepository;
+}
+
+class PhishermanApiError extends Error {
+  status: number;
+  constructor(status: number, message: string) {
+    super(message);
+    this.status = status;
+  }
+}
+
+export function hasPhishermanMasterAPIKey() {
+  return MASTER_API_KEY != null && MASTER_API_KEY !== "";
+}
+
+export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean {
+  return info.classification === "safe";
+}
+
+const leadingSlashRegex = /^\/+/g;
+function trimLeadingSlash(str: string): string {
+  return str.replace(leadingSlashRegex, "");
+}
+
+/**
+ * Make an arbitrary API call to the Phisherman API
+ */
+async function apiCall<T>(
+  method: "GET" | "POST",
+  resource: string,
+  payload?: Record<string, unknown> | null,
+): Promise<T> {
+  if (!hasPhishermanMasterAPIKey()) {
+    throw new Error("Phisherman master API key missing");
+  }
+
+  const url = `${API_URL}/${trimLeadingSlash(resource)}`;
+  const key = `${method} ${url}`;
+
+  if (pendingApiRequests.has(key)) {
+    return pendingApiRequests.get(key)! as Promise<T>;
+  }
+
+  const requestPromise = (async () => {
+    const response = await fetch(url, {
+      method,
+      headers: new Headers({
+        Accept: "application/json",
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${MASTER_API_KEY}`,
+      }),
+      body: payload ? JSON.stringify(payload) : undefined,
+    });
+    const data = await response.json().catch(() => null);
+    if (!response.ok || (data as any)?.success === false) {
+      throw new PhishermanApiError(response.status, (data as any)?.message ?? "");
+    }
+    return data;
+  })();
+  requestPromise.finally(() => {
+    pendingApiRequests.delete(key);
+  });
+  pendingApiRequests.set(key, requestPromise);
+  return requestPromise as Promise<T>;
+}
+
+type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo;
+async function fetchDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
+  // tslint:disable-next-line:no-console
+  console.log(`[PHISHERMAN] Requesting domain information: ${domain}`);
+  const result = await apiCall<Record<string, DomainInfoApiCallResult>>("GET", `/v2/domains/info/${domain}`);
+  const domainInfo = result[domain];
+  if (domainInfo.classification === "unknown") {
+    return null;
+  }
+  return domainInfo;
+}
+
+export async function getPhishermanDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
+  if (pendingDomainInfoChecks.has(domain)) {
+    return pendingDomainInfoChecks.get(domain)!;
+  }
+
+  const promise = (async () => {
+    if (memoryCache.has(domain)) {
+      return memoryCache.get(domain)!.info;
+    }
+
+    const dbCache = getCacheRepository();
+    const existingCachedEntry = await dbCache.findOne({ domain });
+    if (existingCachedEntry) {
+      return existingCachedEntry.data;
+    }
+
+    const freshData = await fetchDomainInfo(domain);
+    const expiryTime =
+      freshData === null
+        ? UNKNOWN_DOMAIN_CACHE_LIFETIME
+        : phishermanDomainIsSafe(freshData)
+        ? SAFE_DOMAIN_CACHE_LIFETIME
+        : DETECTED_DOMAIN_CACHE_LIFETIME;
+    memoryCache.set(domain, {
+      info: freshData,
+      expires: Date.now() + expiryTime,
+    });
+
+    if (freshData) {
+      // Database cache only stores safe/detected domains, not unknown ones
+      await dbCache.insert({
+        domain,
+        data: freshData,
+        expires_at: moment().add(expiryTime, "ms").format(DBDateFormat),
+      });
+    }
+
+    return freshData;
+  })();
+  promise.finally(() => {
+    pendingDomainInfoChecks.delete(domain);
+  });
+  pendingDomainInfoChecks.set(domain, promise);
+
+  return promise;
+}
+
+export async function phishermanApiKeyIsValid(apiKey: string): Promise<boolean> {
+  const keyCache = getKeyCacheRepository();
+  const hash = crypto.createHash("sha256").update(apiKey).digest("hex");
+  const entry = await keyCache.findOne({ hash });
+  if (entry) {
+    return entry.is_valid;
+  }
+
+  const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey });
+
+  await keyCache.insert({
+    hash,
+    is_valid: isValid,
+    expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat),
+  });
+
+  return isValid;
+}
+
+export function trackPhishermanCaughtDomain(apiKey: string, domain: string) {
+  if (!caughtDomainTrackingMap.has(apiKey)) {
+    caughtDomainTrackingMap.set(apiKey, new Map());
+  }
+  const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!;
+  if (!apiKeyMap.has(domain)) {
+    apiKeyMap.set(domain, []);
+  }
+  const timestamps = apiKeyMap.get(domain)!;
+  timestamps.push(Date.now());
+}
+
+export async function reportTrackedDomainsToPhisherman() {
+  const result = {};
+  for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) {
+    result[apiKey] = {};
+    for (const [domain, timestamps] of domains.entries()) {
+      result[apiKey][domain] = timestamps;
+    }
+  }
+
+  if (Object.keys(result).length > 0) {
+    await apiCall("POST", "/v2/phish/caught/bulk", result);
+    caughtDomainTrackingMap = new Map();
+  }
+}
+
+export async function deleteStalePhishermanCacheEntries() {
+  await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
+}
+
+export async function deleteStalePhishermanKeyCacheEntries() {
+  await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
+}

backend/src/data/entities/PhishermanCacheEntry.ts

@@ -0,0 +1,18 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+import { PhishermanDomainInfo } from "../types/phisherman";
+
+@Entity("phisherman_cache")
+export class PhishermanCacheEntry {
+  @Column()
+  @PrimaryColumn()
+  id: number;
+
+  @Column()
+  domain: string;
+
+  @Column("simple-json")
+  data: PhishermanDomainInfo;
+
+  @Column()
+  expires_at: string;
+}

backend/src/data/entities/PhishermanKeyCacheEntry.ts

@@ -0,0 +1,17 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+
+@Entity("phisherman_key_cache")
+export class PhishermanKeyCacheEntry {
+  @Column()
+  @PrimaryColumn()
+  id: number;
+
+  @Column()
+  hash: string;
+
+  @Column()
+  is_valid: boolean;
+
+  @Column()
+  expires_at: string;
+}

backend/src/data/loops/phishermanLoops.ts

@@ -0,0 +1,26 @@
+import { HOURS, MINUTES } from "../../utils";
+import {
+  deleteStalePhishermanCacheEntries,
+  deleteStalePhishermanKeyCacheEntries,
+  reportTrackedDomainsToPhisherman,
+} from "../Phisherman";
+
+const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES;
+const REPORT_LOOP_INTERVAL = 15 * MINUTES;
+
+export async function runPhishermanCacheCleanupLoop() {
+  console.log("[PHISHERMAN] Deleting stale cache entries");
+  await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err));
+
+  console.log("[PHISHERMAN] Deleting stale key cache entries");
+  await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err));
+
+  setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL);
+}
+
+export async function runPhishermanReportingLoop() {
+  console.log("[PHISHERMAN] Reporting tracked domains");
+  await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err));
+
+  setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL);
+}

backend/src/data/types/phisherman.ts

@@ -0,0 +1,32 @@
+export interface PhishermanUnknownDomain {
+  classification: "unknown";
+}
+
+export interface PhishermanDomainInfo {
+  status: string;
+  lastChecked: string;
+  verifiedPhish: boolean;
+  classification: "safe" | "malicious";
+  created: string;
+  firstSeen: string | null;
+  lastSeen: string | null;
+  targetedBrand: string;
+  phishCaught: number;
+  details: PhishermanDomainInfoDetails;
+}
+
+export interface PhishermanDomainInfoDetails {
+  phishTankId: string | null;
+  urlScanId: string;
+  websiteScreenshot: string;
+  ip_address: string;
+  asn: PhishermanDomainInfoAsn;
+  registry: string;
+  country: string;
+}
+
+export interface PhishermanDomainInfoAsn {
+  asn: string;
+  asn_name: string;
+  route: string;
+}

backend/src/index.ts

@@ -33,6 +33,8 @@ import { runSavedMessageCleanupLoop } from "./data/loops/savedMessageCleanupLoop
 import { performance } from "perf_hooks";
 import { setProfiler } from "./profiler";
 import { enableProfiling } from "./utils/easyProfiler";
+import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./data/loops/phishermanLoops";
+import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
 
 if (!process.env.KEY) {
   // tslint:disable-next-line:no-console
@@ -363,6 +365,13 @@ connect().then(async () => {
     runExpiredArchiveDeletionLoop();
     await sleep(10 * SECONDS);
     runSavedMessageCleanupLoop();
+
+    if (hasPhishermanMasterAPIKey()) {
+      await sleep(10 * SECONDS);
+      runPhishermanCacheCleanupLoop();
+      await sleep(10 * SECONDS);
+      runPhishermanReportingLoop();
+    }
   });
 
   setProfiler(bot.profiler);

backend/src/migrations/1634563901575-CreatePhishermanCacheTable.ts

@@ -0,0 +1,38 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreatePhishermanCacheTable1634563901575 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: "phisherman_cache",
+        columns: [
+          {
+            name: "id",
+            type: "int",
+            isPrimary: true,
+            isGenerated: true,
+            generationStrategy: "increment",
+          },
+          {
+            name: "domain",
+            type: "varchar",
+            length: "255",
+            isUnique: true,
+          },
+          {
+            name: "data",
+            type: "text",
+          },
+          {
+            name: "expires_at",
+            type: "datetime",
+          },
+        ],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropTable("phisherman_cache");
+  }
+}

backend/src/migrations/1635596150234-CreatePhishermanKeyCacheTable.ts

@@ -0,0 +1,38 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreatePhishermanKeyCacheTable1635596150234 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: "phisherman_key_cache",
+        columns: [
+          {
+            name: "id",
+            type: "int",
+            isPrimary: true,
+            isGenerated: true,
+            generationStrategy: "increment",
+          },
+          {
+            name: "hash",
+            type: "varchar",
+            length: "255",
+            isUnique: true,
+          },
+          {
+            name: "is_valid",
+            type: "tinyint",
+          },
+          {
+            name: "expires_at",
+            type: "datetime",
+          },
+        ],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropTable("phisherman_key_cache");
+  }
+}

backend/src/plugins/Automod/AutomodPlugin.ts

@@ -30,6 +30,7 @@ import { clearOldRecentSpam } from "./functions/clearOldRecentSpam";
 import { pluginInfo } from "./info";
 import { availableTriggers } from "./triggers/availableTriggers";
 import { AutomodPluginType, ConfigSchema } from "./types";
+import { PhishermanPlugin } from "../Phisherman/PhishermanPlugin";
 
 const defaultOptions = {
   config: {
@@ -183,6 +184,7 @@ export const AutomodPlugin = zeppelinGuildPlugin<AutomodPluginType>()({
     ModActionsPlugin,
     MutesPlugin,
     CountersPlugin,
+    PhishermanPlugin,
   ],
 
   configSchema: ConfigSchema,

backend/src/plugins/Automod/triggers/matchLinks.ts

@@ -9,10 +9,13 @@ import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions
 import { automodTrigger } from "../helpers";
 import { mergeRegexes } from "../../../utils/mergeRegexes";
 import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex";
+import { PhishermanPlugin } from "../../Phisherman/PhishermanPlugin";
+import { phishermanDomainIsSafe } from "../../../data/Phisherman";
 
 interface MatchResultType {
   type: MatchableTextType;
   link: string;
+  details?: string;
 }
 
 const regexCache = new WeakMap<any, RegExp[]>();
@@ -28,6 +31,12 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
     exclude_words: tNullable(t.array(t.string)),
     include_regex: tNullable(t.array(TRegex)),
     exclude_regex: tNullable(t.array(TRegex)),
+    phisherman: tNullable(
+      t.type({
+        include_suspected: tNullable(t.boolean),
+        include_verified: tNullable(t.boolean),
+      }),
+    ),
     only_real_links: t.boolean,
     match_messages: t.boolean,
     match_embeds: t.boolean,
@@ -150,6 +159,25 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
             }
           }
         }
+
+        if (trigger.phisherman) {
+          const phishermanResult = await pluginData.getPlugin(PhishermanPlugin).getDomainInfo(normalizedHostname);
+          if (phishermanResult != null && !phishermanDomainIsSafe(phishermanResult)) {
+            if (
+              (trigger.phisherman.include_suspected && !phishermanResult.verifiedPhish) ||
+              (trigger.phisherman.include_verified && phishermanResult.verifiedPhish)
+            ) {
+              const suspectedVerified = phishermanResult.verifiedPhish ? "verified" : "suspected";
+              return {
+                extra: {
+                  type,
+                  link: link.input,
+                  details: `using Phisherman (${suspectedVerified})`,
+                },
+              };
+            }
+          }
+        }
       }
     }
 
@@ -158,6 +186,11 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
 
   renderMatchInformation({ pluginData, contexts, matchResult }) {
     const partialSummary = getTextMatchPartialSummary(pluginData, matchResult.extra.type, contexts[0]);
-    return `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\` in ${partialSummary}`;
+    let information = `Matched link \`${Util.escapeInlineCode(matchResult.extra.link)}\``;
+    if (matchResult.extra.details) {
+      information += ` ${matchResult.extra.details}`;
+    }
+    information += ` in ${partialSummary}`;
+    return information;
   },
 });

backend/src/plugins/Phisherman/PhishermanPlugin.ts

@@ -0,0 +1,49 @@
+import { PluginOptions, typedGuildCommand } from "knub";
+import { GuildPingableRoles } from "../../data/GuildPingableRoles";
+import { zeppelinGuildPlugin } from "../ZeppelinPluginBlueprint";
+import { ConfigSchema, PhishermanPluginType } from "./types";
+import {
+  getPhishermanDomainInfo,
+  hasPhishermanMasterAPIKey,
+  phishermanApiKeyIsValid,
+  reportTrackedDomainsToPhisherman,
+} from "../../data/Phisherman";
+import { mapToPublicFn } from "../../pluginUtils";
+import { getDomainInfo } from "./functions/getDomainInfo";
+import { pluginInfo } from "./info";
+
+const defaultOptions: PluginOptions<PhishermanPluginType> = {
+  config: {
+    api_key: null,
+  },
+  overrides: [],
+};
+
+export const PhishermanPlugin = zeppelinGuildPlugin<PhishermanPluginType>()({
+  name: "phisherman",
+  showInDocs: true,
+  info: pluginInfo,
+
+  configSchema: ConfigSchema,
+  defaultOptions,
+
+  // prettier-ignore
+  public: {
+    getDomainInfo: mapToPublicFn(getDomainInfo),
+  },
+
+  async beforeLoad(pluginData) {
+    pluginData.state.validApiKey = null;
+
+    if (!hasPhishermanMasterAPIKey()) {
+      // tslint:disable-next-line:no-console
+      console.warn("Could not load Phisherman plugin: master API key is missing");
+      return;
+    }
+
+    const apiKey = pluginData.config.get().api_key;
+    if (apiKey && (await phishermanApiKeyIsValid(apiKey).catch(() => false))) {
+      pluginData.state.validApiKey = apiKey;
+    }
+  },
+});

backend/src/plugins/Phisherman/functions/getDomainInfo.ts

@@ -0,0 +1,20 @@
+import { GuildPluginData } from "knub";
+import { PhishermanPluginType } from "../types";
+import { PhishermanDomainInfo } from "../../../data/types/phisherman";
+import { getPhishermanDomainInfo, phishermanDomainIsSafe, trackPhishermanCaughtDomain } from "../../../data/Phisherman";
+
+export async function getDomainInfo(
+  pluginData: GuildPluginData<PhishermanPluginType>,
+  domain: string,
+): Promise<PhishermanDomainInfo | null> {
+  if (!pluginData.state.validApiKey) {
+    return null;
+  }
+
+  const info = await getPhishermanDomainInfo(domain).catch(() => null);
+  if (info != null && !phishermanDomainIsSafe(info)) {
+    trackPhishermanCaughtDomain(pluginData.state.validApiKey, domain);
+  }
+
+  return info;
+}

backend/src/plugins/Phisherman/info.ts

@@ -0,0 +1,41 @@
+import { trimPluginDescription } from "../../utils";
+import { ZeppelinGuildPluginBlueprint } from "../ZeppelinPluginBlueprint";
+
+export const pluginInfo: ZeppelinGuildPluginBlueprint["info"] = {
+  prettyName: "Phisherman",
+  description: trimPluginDescription(`
+    Match scam/phishing links using the Phisherman API. See https://phisherman.gg/ for more details!
+  `),
+  configurationGuide: trimPluginDescription(`
+    ### Getting started
+    To get started, request an API key for Phisherman following the instructions at https://docs.phisherman.gg/#/api/getting-started?id=requesting-api-access.
+    Then, add the api key to the plugin's config:
+    
+    ~~~yml
+    phisherman:
+      config:
+        api_key: "your key here"
+    ~~~
+    
+    ### Note
+    When using Phisherman features in Zeppelin, Zeppelin reports statistics about checked links back to Phisherman. This only includes the domain (e.g. zeppelin.gg), not the full link.
+    
+    ### Usage with Automod
+    Once you have configured the Phisherman plugin, you are ready to use it with automod. Currently, Phisherman is available as an option in the \`match_links\` plugin:
+    
+    ~~~yml
+    automod:
+      config:
+        rules:
+          # Clean any scam links detected by Phisherman
+          filter_scam_links:
+            triggers:
+              - match_links:
+                  phisherman:
+                    include_suspected: true # It's recommended to keep this enabled to catch new scam domains quickly
+                    include_verified: true
+            actions:
+              clean: true
+    ~~~
+  `),
+};

backend/src/plugins/Phisherman/types.ts

@@ -0,0 +1,16 @@
+import * as t from "io-ts";
+import { BasePluginType } from "knub";
+import { tNullable } from "../../utils";
+
+export const ConfigSchema = t.type({
+  api_key: tNullable(t.string),
+});
+export type TConfigSchema = t.TypeOf<typeof ConfigSchema>;
+
+export interface PhishermanPluginType extends BasePluginType {
+  config: TConfigSchema;
+
+  state: {
+    validApiKey: string | null;
+  };
+}

backend/src/plugins/availablePlugins.ts

@@ -34,6 +34,7 @@ import { UsernameSaverPlugin } from "./UsernameSaver/UsernameSaverPlugin";
 import { UtilityPlugin } from "./Utility/UtilityPlugin";
 import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin";
 import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint";
+import { PhishermanPlugin } from "./Phisherman/PhishermanPlugin";
 
 // prettier-ignore
 export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
@@ -69,6 +70,7 @@ export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
   TimeAndDatePlugin,
   CountersPlugin,
   ContextMenuPlugin,
+  PhishermanPlugin,
 ];
 
 // prettier-ignore