mirror of
https://github.com/ZeppelinBot/Zeppelin.git
synced 2025-06-07 08:05:01 +00:00
feat: replace Phisherman with FishFish
This commit is contained in:
Dragory
parent
bf08cade15
commit
8c2058821f
13 changed files with 202 additions and 475 deletions
|
|
@ -22,7 +22,7 @@ STAFF=
|
||||||
DEFAULT_ALLOWED_SERVERS=
|
DEFAULT_ALLOWED_SERVERS=
|
||||||
|
|
||||||
# Only required if relevant feature is used
|
# Only required if relevant feature is used
|
||||||
#PHISHERMAN_API_KEY=
|
#FISHFISH_API_KEY=
|
||||||
|
|
||||||
|
|
||||||
# ==========================
|
# ==========================
|
||||||
|
|
|
||||||
173
backend/src/data/FishFish.ts
Normal file
173
backend/src/data/FishFish.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
||||||
|
import z from "zod/v4";
|
||||||
|
import { env } from "../env.js";
|
||||||
|
import { HOURS, MINUTES, SECONDS } from "../utils.js";
|
||||||
|
|
||||||
|
const API_ROOT = "https://api.fishfish.gg/v1";
|
||||||
|
|
||||||
|
const zDomainCategory = z.literal(["safe", "malware", "phishing"]);
|
||||||
|
|
||||||
|
const zDomain = z.object({
|
||||||
|
name: z.string(),
|
||||||
|
category: zDomainCategory,
|
||||||
|
description: z.string(),
|
||||||
|
added: z.number(),
|
||||||
|
checked: z.number(),
|
||||||
|
});
|
||||||
|
export type FishFishDomain = z.output<typeof zDomain>;
|
||||||
|
|
||||||
|
const FULL_REFRESH_INTERVAL = 6 * HOURS;
|
||||||
|
const domains = new Map<string, FishFishDomain>();
|
||||||
|
|
||||||
|
let sessionTokenPromise: Promise<string> | null = null;
|
||||||
|
|
||||||
|
const WS_RECONNECT_DELAY = 30 * SECONDS;
|
||||||
|
let updatesWs: WebSocket | null = null;
|
||||||
|
|
||||||
|
export class FishFishError extends Error {}
|
||||||
|
|
||||||
|
const zTokenResponse = z.object({
|
||||||
|
expires: z.number(),
|
||||||
|
token: z.string(),
|
||||||
|
});
|
||||||
|
|
||||||
|
async function getSessionToken(): Promise<string> {
|
||||||
|
if (sessionTokenPromise) {
|
||||||
|
return sessionTokenPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
const apiKey = env.FISHFISH_API_KEY;
|
||||||
|
if (!apiKey) {
|
||||||
|
throw new FishFishError("FISHFISH_API_KEY is missing");
|
||||||
|
}
|
||||||
|
|
||||||
|
sessionTokenPromise = (async () => {
|
||||||
|
const response = await fetch(`${API_ROOT}/users/@me/tokens`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
Authorization: apiKey,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new FishFishError(`Failed to get session token: ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const parseResult = zTokenResponse.safeParse(await response.json());
|
||||||
|
if (!parseResult.success) {
|
||||||
|
throw new FishFishError(`Parse error when fetching session token: ${parseResult.error.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeUntilExpiry = Date.now() - parseResult.data.expires * 1000;
|
||||||
|
setTimeout(() => {
|
||||||
|
sessionTokenPromise = null;
|
||||||
|
}, timeUntilExpiry - 1 * MINUTES); // Subtract a minute to ensure we refresh before expiry
|
||||||
|
|
||||||
|
return parseResult.data.token;
|
||||||
|
})();
|
||||||
|
sessionTokenPromise.catch((err) => {
|
||||||
|
sessionTokenPromise = null;
|
||||||
|
throw err;
|
||||||
|
});
|
||||||
|
|
||||||
|
return sessionTokenPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fishFishApiCall(method: string, path: string, query: Record<string, string> = {}): Promise<unknown> {
|
||||||
|
const sessionToken = await getSessionToken();
|
||||||
|
const queryParams = new URLSearchParams(query);
|
||||||
|
const response = await fetch(`https://api.fishfish.gg/v1/${path}?${queryParams}`, {
|
||||||
|
method,
|
||||||
|
headers: {
|
||||||
|
Authorization: sessionToken,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new FishFishError(`FishFish API call failed: ${response.status} ${response.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function subscribeToFishFishUpdates(): Promise<void> {
|
||||||
|
if (updatesWs) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const sessionToken = await getSessionToken();
|
||||||
|
console.log("[FISHFISH] Connecting to WebSocket for real-time updates");
|
||||||
|
updatesWs = new WebSocket("wss://api.fishfish.gg/v1/stream", {
|
||||||
|
headers: {
|
||||||
|
Authorization: sessionToken,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
updatesWs.addEventListener("open", () => {
|
||||||
|
console.log("[FISHFISH] WebSocket connection established");
|
||||||
|
});
|
||||||
|
updatesWs.addEventListener("message", (event) => {
|
||||||
|
console.log("[FISHFISH] ws update:", event.data);
|
||||||
|
});
|
||||||
|
updatesWs.addEventListener("error", (error) => {
|
||||||
|
console.error(`[FISHFISH] WebSocket error: ${error.message}`);
|
||||||
|
});
|
||||||
|
updatesWs.addEventListener("close", () => {
|
||||||
|
console.log("[FISHFISH] WebSocket connection closed, reconnecting after delay");
|
||||||
|
updatesWs = null;
|
||||||
|
setTimeout(() => {
|
||||||
|
subscribeToFishFishUpdates();
|
||||||
|
}, WS_RECONNECT_DELAY);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function refreshFishFishDomains() {
|
||||||
|
const rawData = await fishFishApiCall("GET", "domains", { full: "true" });
|
||||||
|
const parseResult = z.array(zDomain).safeParse(rawData);
|
||||||
|
if (!parseResult.success) {
|
||||||
|
throw new FishFishError(`Parse error when refreshing domains: ${parseResult.error.message}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
domains.clear();
|
||||||
|
for (const domain of parseResult.data) {
|
||||||
|
domains.set(domain.name, domain);
|
||||||
|
}
|
||||||
|
|
||||||
|
domains.set("malware-link.test.zeppelin.gg", {
|
||||||
|
name: "malware-link.test.zeppelin.gg",
|
||||||
|
category: "malware",
|
||||||
|
description: "",
|
||||||
|
added: Date.now(),
|
||||||
|
checked: Date.now(),
|
||||||
|
});
|
||||||
|
domains.set("phishing-link.test.zeppelin.gg", {
|
||||||
|
name: "phishing-link.test.zeppelin.gg",
|
||||||
|
category: "phishing",
|
||||||
|
description: "",
|
||||||
|
added: Date.now(),
|
||||||
|
checked: Date.now(),
|
||||||
|
});
|
||||||
|
domains.set("safe-link.test.zeppelin.gg", {
|
||||||
|
name: "safe-link.test.zeppelin.gg",
|
||||||
|
category: "safe",
|
||||||
|
description: "",
|
||||||
|
added: Date.now(),
|
||||||
|
checked: Date.now(),
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log("[FISHFISH] Refreshed FishFish domains, total count:", domains.size);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function initFishFish() {
|
||||||
|
if (!env.FISHFISH_API_KEY) {
|
||||||
|
console.warn("[FISHFISH] FISHFISH_API_KEY is not set, FishFish functionality will be disabled.");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
await refreshFishFishDomains();
|
||||||
|
void subscribeToFishFishUpdates();
|
||||||
|
setInterval(() => refreshFishFishDomains(), FULL_REFRESH_INTERVAL);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getFishFishDomain(domain: string): FishFishDomain | undefined {
|
||||||
|
return domains.get(domain.toLowerCase());
|
||||||
|
}
|
||||||
|
|
@ -1,253 +0,0 @@
|
||||||
import crypto from "crypto";
|
|
||||||
import moment from "moment-timezone";
|
|
||||||
import { Repository } from "typeorm";
|
|
||||||
import { env } from "../env.js";
|
|
||||||
import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils.js";
|
|
||||||
import { dataSource } from "./dataSource.js";
|
|
||||||
import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry.js";
|
|
||||||
import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry.js";
|
|
||||||
import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman.js";
|
|
||||||
|
|
||||||
const API_URL = "https://api.phisherman.gg";
|
|
||||||
const MASTER_API_KEY = env.PHISHERMAN_API_KEY;
|
|
||||||
|
|
||||||
let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
|
|
||||||
|
|
||||||
const pendingApiRequests: Map<string, Promise<unknown>> = new Map();
|
|
||||||
const pendingDomainInfoChecks: Map<string, Promise<PhishermanDomainInfo | null>> = new Map();
|
|
||||||
|
|
||||||
type MemoryCacheEntry = {
|
|
||||||
info: PhishermanDomainInfo | null;
|
|
||||||
expires: number;
|
|
||||||
};
|
|
||||||
const memoryCache: Map<string, MemoryCacheEntry> = new Map();
|
|
||||||
|
|
||||||
setInterval(() => {
|
|
||||||
const now = Date.now();
|
|
||||||
for (const [key, entry] of memoryCache.entries()) {
|
|
||||||
if (entry.expires <= now) {
|
|
||||||
memoryCache.delete(key);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, 2 * MINUTES);
|
|
||||||
|
|
||||||
const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES;
|
|
||||||
const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES;
|
|
||||||
const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS;
|
|
||||||
|
|
||||||
const KEY_VALIDITY_LIFETIME = 24 * HOURS;
|
|
||||||
|
|
||||||
let cacheRepository: Repository<PhishermanCacheEntry> | null = null;
|
|
||||||
function getCacheRepository(): Repository<PhishermanCacheEntry> {
|
|
||||||
if (cacheRepository == null) {
|
|
||||||
cacheRepository = dataSource.getRepository(PhishermanCacheEntry);
|
|
||||||
}
|
|
||||||
return cacheRepository;
|
|
||||||
}
|
|
||||||
|
|
||||||
let keyCacheRepository: Repository<PhishermanKeyCacheEntry> | null = null;
|
|
||||||
function getKeyCacheRepository(): Repository<PhishermanKeyCacheEntry> {
|
|
||||||
if (keyCacheRepository == null) {
|
|
||||||
keyCacheRepository = dataSource.getRepository(PhishermanKeyCacheEntry);
|
|
||||||
}
|
|
||||||
return keyCacheRepository;
|
|
||||||
}
|
|
||||||
|
|
||||||
class PhishermanApiError extends Error {
|
|
||||||
method: string;
|
|
||||||
url: string;
|
|
||||||
status: number;
|
|
||||||
|
|
||||||
constructor(method: string, url: string, status: number, message: string) {
|
|
||||||
super(message);
|
|
||||||
this.method = method;
|
|
||||||
this.url = url;
|
|
||||||
this.status = status;
|
|
||||||
}
|
|
||||||
|
|
||||||
toString() {
|
|
||||||
return `Error ${this.status} in ${this.method} ${this.url}: ${this.message}`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function hasPhishermanMasterAPIKey() {
|
|
||||||
return MASTER_API_KEY != null && MASTER_API_KEY !== "";
|
|
||||||
}
|
|
||||||
|
|
||||||
export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean {
|
|
||||||
return info.classification === "safe";
|
|
||||||
}
|
|
||||||
|
|
||||||
const leadingSlashRegex = /^\/+/g;
|
|
||||||
function trimLeadingSlash(str: string): string {
|
|
||||||
return str.replace(leadingSlashRegex, "");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Make an arbitrary API call to the Phisherman API
|
|
||||||
*/
|
|
||||||
async function apiCall<T>(
|
|
||||||
method: "GET" | "POST",
|
|
||||||
resource: string,
|
|
||||||
payload?: Record<string, unknown> | null,
|
|
||||||
): Promise<T> {
|
|
||||||
if (!hasPhishermanMasterAPIKey()) {
|
|
||||||
throw new Error("Phisherman master API key missing");
|
|
||||||
}
|
|
||||||
|
|
||||||
const url = `${API_URL}/${trimLeadingSlash(resource)}`;
|
|
||||||
const key = `${method} ${url}`;
|
|
||||||
|
|
||||||
if (pendingApiRequests.has(key)) {
|
|
||||||
return pendingApiRequests.get(key)! as Promise<T>;
|
|
||||||
}
|
|
||||||
|
|
||||||
let requestPromise = (async () => {
|
|
||||||
const response = await fetch(url, {
|
|
||||||
method,
|
|
||||||
headers: new Headers({
|
|
||||||
Accept: "application/json",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
Authorization: `Bearer ${MASTER_API_KEY}`,
|
|
||||||
}),
|
|
||||||
body: payload ? JSON.stringify(payload) : undefined,
|
|
||||||
});
|
|
||||||
const data = await response.json().catch(() => null);
|
|
||||||
if (!response.ok || (data as any)?.success === false) {
|
|
||||||
throw new PhishermanApiError(method, url, response.status, (data as any)?.message ?? "");
|
|
||||||
}
|
|
||||||
return data;
|
|
||||||
})();
|
|
||||||
requestPromise = requestPromise.finally(() => {
|
|
||||||
pendingApiRequests.delete(key);
|
|
||||||
});
|
|
||||||
pendingApiRequests.set(key, requestPromise);
|
|
||||||
return requestPromise as Promise<T>;
|
|
||||||
}
|
|
||||||
|
|
||||||
type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo;
|
|
||||||
async function fetchDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
|
||||||
// tslint:disable-next-line:no-console
|
|
||||||
console.log(`[PHISHERMAN] Requesting domain information: ${domain}`);
|
|
||||||
const result = await apiCall<Record<string, DomainInfoApiCallResult>>("GET", `/v2/domains/info/${domain}`);
|
|
||||||
const firstKey = Object.keys(result)[0];
|
|
||||||
const domainInfo = firstKey ? result[firstKey] : null;
|
|
||||||
if (!domainInfo) {
|
|
||||||
// tslint:disable-next-line:no-console
|
|
||||||
console.warn(`Unexpected Phisherman API response for ${domain}:`, result);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
if (domainInfo.classification === "unknown") {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return domainInfo;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getPhishermanDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
|
||||||
if (pendingDomainInfoChecks.has(domain)) {
|
|
||||||
return pendingDomainInfoChecks.get(domain)!;
|
|
||||||
}
|
|
||||||
|
|
||||||
let promise = (async () => {
|
|
||||||
if (memoryCache.has(domain)) {
|
|
||||||
return memoryCache.get(domain)!.info;
|
|
||||||
}
|
|
||||||
|
|
||||||
const dbCache = getCacheRepository();
|
|
||||||
const existingCachedEntry = await dbCache.findOne({
|
|
||||||
where: { domain },
|
|
||||||
});
|
|
||||||
if (existingCachedEntry) {
|
|
||||||
return existingCachedEntry.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
const freshData = await fetchDomainInfo(domain);
|
|
||||||
const expiryTime =
|
|
||||||
freshData === null
|
|
||||||
? UNKNOWN_DOMAIN_CACHE_LIFETIME
|
|
||||||
: phishermanDomainIsSafe(freshData)
|
|
||||||
? SAFE_DOMAIN_CACHE_LIFETIME
|
|
||||||
: DETECTED_DOMAIN_CACHE_LIFETIME;
|
|
||||||
memoryCache.set(domain, {
|
|
||||||
info: freshData,
|
|
||||||
expires: Date.now() + expiryTime,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (freshData) {
|
|
||||||
// Database cache only stores safe/detected domains, not unknown ones
|
|
||||||
await dbCache.insert({
|
|
||||||
domain,
|
|
||||||
data: freshData,
|
|
||||||
expires_at: moment().add(expiryTime, "ms").format(DBDateFormat),
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return freshData;
|
|
||||||
})();
|
|
||||||
promise = promise.finally(() => {
|
|
||||||
pendingDomainInfoChecks.delete(domain);
|
|
||||||
});
|
|
||||||
pendingDomainInfoChecks.set(domain, promise);
|
|
||||||
|
|
||||||
return promise;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function phishermanApiKeyIsValid(apiKey: string): Promise<boolean> {
|
|
||||||
if (apiKey === MASTER_API_KEY) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
const keyCache = getKeyCacheRepository();
|
|
||||||
const hash = crypto.createHash("sha256").update(apiKey).digest("hex");
|
|
||||||
const entry = await keyCache.findOne({
|
|
||||||
where: { hash },
|
|
||||||
});
|
|
||||||
if (entry) {
|
|
||||||
return entry.is_valid;
|
|
||||||
}
|
|
||||||
|
|
||||||
const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey });
|
|
||||||
|
|
||||||
await keyCache.insert({
|
|
||||||
hash,
|
|
||||||
is_valid: isValid,
|
|
||||||
expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat),
|
|
||||||
});
|
|
||||||
|
|
||||||
return isValid;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function trackPhishermanCaughtDomain(apiKey: string, domain: string) {
|
|
||||||
if (!caughtDomainTrackingMap.has(apiKey)) {
|
|
||||||
caughtDomainTrackingMap.set(apiKey, new Map());
|
|
||||||
}
|
|
||||||
const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!;
|
|
||||||
if (!apiKeyMap.has(domain)) {
|
|
||||||
apiKeyMap.set(domain, []);
|
|
||||||
}
|
|
||||||
const timestamps = apiKeyMap.get(domain)!;
|
|
||||||
timestamps.push(Date.now());
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function reportTrackedDomainsToPhisherman() {
|
|
||||||
const result = {};
|
|
||||||
for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) {
|
|
||||||
result[apiKey] = {};
|
|
||||||
for (const [domain, timestamps] of domains.entries()) {
|
|
||||||
result[apiKey][domain] = timestamps;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Object.keys(result).length > 0) {
|
|
||||||
await apiCall("POST", "/v2/phish/caught/bulk", result);
|
|
||||||
caughtDomainTrackingMap = new Map();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function deleteStalePhishermanCacheEntries() {
|
|
||||||
await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function deleteStalePhishermanKeyCacheEntries() {
|
|
||||||
await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
|
||||||
}
|
|
||||||
|
|
@ -1,18 +0,0 @@
|
||||||
import { Column, Entity, PrimaryColumn } from "typeorm";
|
|
||||||
import { PhishermanDomainInfo } from "../types/phisherman.js";
|
|
||||||
|
|
||||||
@Entity("phisherman_cache")
|
|
||||||
export class PhishermanCacheEntry {
|
|
||||||
@Column()
|
|
||||||
@PrimaryColumn()
|
|
||||||
id: number;
|
|
||||||
|
|
||||||
@Column()
|
|
||||||
domain: string;
|
|
||||||
|
|
||||||
@Column("simple-json")
|
|
||||||
data: PhishermanDomainInfo;
|
|
||||||
|
|
||||||
@Column()
|
|
||||||
expires_at: string;
|
|
||||||
}
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
import { Column, Entity, PrimaryColumn } from "typeorm";
|
|
||||||
|
|
||||||
@Entity("phisherman_key_cache")
|
|
||||||
export class PhishermanKeyCacheEntry {
|
|
||||||
@Column()
|
|
||||||
@PrimaryColumn()
|
|
||||||
id: number;
|
|
||||||
|
|
||||||
@Column()
|
|
||||||
hash: string;
|
|
||||||
|
|
||||||
@Column()
|
|
||||||
is_valid: boolean;
|
|
||||||
|
|
||||||
@Column()
|
|
||||||
expires_at: string;
|
|
||||||
}
|
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
// tslint:disable:no-console
|
|
||||||
|
|
||||||
import { MINUTES } from "../../utils.js";
|
|
||||||
import {
|
|
||||||
deleteStalePhishermanCacheEntries,
|
|
||||||
deleteStalePhishermanKeyCacheEntries,
|
|
||||||
reportTrackedDomainsToPhisherman,
|
|
||||||
} from "../Phisherman.js";
|
|
||||||
|
|
||||||
const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES;
|
|
||||||
const REPORT_LOOP_INTERVAL = 15 * MINUTES;
|
|
||||||
|
|
||||||
export async function runPhishermanCacheCleanupLoop() {
|
|
||||||
console.log("[PHISHERMAN] Deleting stale cache entries");
|
|
||||||
await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err));
|
|
||||||
|
|
||||||
console.log("[PHISHERMAN] Deleting stale key cache entries");
|
|
||||||
await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err));
|
|
||||||
|
|
||||||
setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function runPhishermanReportingLoop() {
|
|
||||||
console.log("[PHISHERMAN] Reporting tracked domains");
|
|
||||||
await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err));
|
|
||||||
|
|
||||||
setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL);
|
|
||||||
}
|
|
||||||
|
|
@ -1,32 +0,0 @@
|
||||||
export interface PhishermanUnknownDomain {
|
|
||||||
classification: "unknown";
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PhishermanDomainInfo {
|
|
||||||
status: string;
|
|
||||||
lastChecked: string;
|
|
||||||
verifiedPhish: boolean;
|
|
||||||
classification: "safe" | "malicious";
|
|
||||||
created: string;
|
|
||||||
firstSeen: string | null;
|
|
||||||
lastSeen: string | null;
|
|
||||||
targetedBrand: string;
|
|
||||||
phishCaught: number;
|
|
||||||
details: PhishermanDomainInfoDetails;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PhishermanDomainInfoDetails {
|
|
||||||
phishTankId: string | null;
|
|
||||||
urlScanId: string;
|
|
||||||
websiteScreenshot: string;
|
|
||||||
ip_address: string;
|
|
||||||
asn: PhishermanDomainInfoAsn;
|
|
||||||
registry: string;
|
|
||||||
country: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PhishermanDomainInfoAsn {
|
|
||||||
asn: string;
|
|
||||||
asn_name: string;
|
|
||||||
route: string;
|
|
||||||
}
|
|
||||||
|
|
@ -21,6 +21,7 @@ import { RecoverablePluginError } from "./RecoverablePluginError.js";
|
||||||
import { SimpleError } from "./SimpleError.js";
|
import { SimpleError } from "./SimpleError.js";
|
||||||
import { AllowedGuilds } from "./data/AllowedGuilds.js";
|
import { AllowedGuilds } from "./data/AllowedGuilds.js";
|
||||||
import { Configs } from "./data/Configs.js";
|
import { Configs } from "./data/Configs.js";
|
||||||
|
import { FishFishError, initFishFish } from "./data/FishFish.js";
|
||||||
import { GuildLogs } from "./data/GuildLogs.js";
|
import { GuildLogs } from "./data/GuildLogs.js";
|
||||||
import { LogType } from "./data/LogType.js";
|
import { LogType } from "./data/LogType.js";
|
||||||
import { hasPhishermanMasterAPIKey } from "./data/Phisherman.js";
|
import { hasPhishermanMasterAPIKey } from "./data/Phisherman.js";
|
||||||
|
|
@ -142,6 +143,12 @@ function errorHandler(err) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (err instanceof FishFishError) {
|
||||||
|
// FishFish errors are not critical, so we just log them
|
||||||
|
console.error(`[FISHFISH] ${err.message}`);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
// tslint:disable:no-console
|
// tslint:disable:no-console
|
||||||
console.error(err);
|
console.error(err);
|
||||||
|
|
||||||
|
|
@ -402,6 +409,8 @@ connect().then(async () => {
|
||||||
enableProfiling();
|
enableProfiling();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
initFishFish();
|
||||||
|
|
||||||
runExpiringMutesLoop();
|
runExpiringMutesLoop();
|
||||||
await sleep(10 * SECONDS);
|
await sleep(10 * SECONDS);
|
||||||
runExpiringTempbansLoop();
|
runExpiringTempbansLoop();
|
||||||
|
|
@ -419,13 +428,6 @@ connect().then(async () => {
|
||||||
runExpiredMemberCacheDeletionLoop();
|
runExpiredMemberCacheDeletionLoop();
|
||||||
await sleep(10 * SECONDS);
|
await sleep(10 * SECONDS);
|
||||||
runMemberCacheDeletionLoop();
|
runMemberCacheDeletionLoop();
|
||||||
|
|
||||||
if (hasPhishermanMasterAPIKey()) {
|
|
||||||
await sleep(10 * SECONDS);
|
|
||||||
runPhishermanCacheCleanupLoop();
|
|
||||||
await sleep(10 * SECONDS);
|
|
||||||
runPhishermanReportingLoop();
|
|
||||||
}
|
|
||||||
});
|
});
|
||||||
|
|
||||||
let lowestGlobalRemaining = Infinity;
|
let lowestGlobalRemaining = Infinity;
|
||||||
|
|
|
||||||
|
|
@ -1,11 +1,10 @@
|
||||||
import { escapeInlineCode } from "discord.js";
|
import { escapeInlineCode } from "discord.js";
|
||||||
import z from "zod/v4";
|
import z from "zod/v4";
|
||||||
import { allowTimeout } from "../../../RegExpRunner.js";
|
import { allowTimeout } from "../../../RegExpRunner.js";
|
||||||
import { phishermanDomainIsSafe } from "../../../data/Phisherman.js";
|
import { getFishFishDomain } from "../../../data/FishFish.js";
|
||||||
import { getUrlsInString, zRegex } from "../../../utils.js";
|
import { getUrlsInString, zRegex } from "../../../utils.js";
|
||||||
import { mergeRegexes } from "../../../utils/mergeRegexes.js";
|
import { mergeRegexes } from "../../../utils/mergeRegexes.js";
|
||||||
import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex.js";
|
import { mergeWordsIntoRegex } from "../../../utils/mergeWordsIntoRegex.js";
|
||||||
import { PhishermanPlugin } from "../../Phisherman/PhishermanPlugin.js";
|
|
||||||
import { getTextMatchPartialSummary } from "../functions/getTextMatchPartialSummary.js";
|
import { getTextMatchPartialSummary } from "../functions/getTextMatchPartialSummary.js";
|
||||||
import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions/matchMultipleTextTypesOnMessage.js";
|
import { MatchableTextType, matchMultipleTextTypesOnMessage } from "../functions/matchMultipleTextTypesOnMessage.js";
|
||||||
import { automodTrigger } from "../helpers.js";
|
import { automodTrigger } from "../helpers.js";
|
||||||
|
|
@ -40,6 +39,7 @@ const configSchema = z.strictObject({
|
||||||
include_verified: z.boolean().optional(),
|
include_verified: z.boolean().optional(),
|
||||||
})
|
})
|
||||||
.optional(),
|
.optional(),
|
||||||
|
include_malicious: z.boolean().default(false),
|
||||||
only_real_links: z.boolean().default(true),
|
only_real_links: z.boolean().default(true),
|
||||||
match_messages: z.boolean().default(true),
|
match_messages: z.boolean().default(true),
|
||||||
match_embeds: z.boolean().default(true),
|
match_embeds: z.boolean().default(true),
|
||||||
|
|
@ -155,26 +155,22 @@ export const MatchLinksTrigger = automodTrigger<MatchResultType>()({
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (trigger.phisherman) {
|
const includeMalicious =
|
||||||
const phishermanResult = await pluginData.getPlugin(PhishermanPlugin).getDomainInfo(normalizedHostname);
|
trigger.include_malicious || trigger.phisherman?.include_suspected || trigger.phisherman?.include_verified;
|
||||||
if (phishermanResult != null && !phishermanDomainIsSafe(phishermanResult)) {
|
if (includeMalicious) {
|
||||||
if (
|
const domainInfo = getFishFishDomain(normalizedHostname);
|
||||||
(trigger.phisherman.include_suspected && !phishermanResult.verifiedPhish) ||
|
if (domainInfo && domainInfo.category !== "safe") {
|
||||||
(trigger.phisherman.include_verified && phishermanResult.verifiedPhish)
|
|
||||||
) {
|
|
||||||
const suspectedVerified = phishermanResult.verifiedPhish ? "verified" : "suspected";
|
|
||||||
return {
|
return {
|
||||||
extra: {
|
extra: {
|
||||||
type,
|
type,
|
||||||
link: link.input,
|
link: link.input,
|
||||||
details: `using Phisherman (${suspectedVerified})`,
|
details: `(known ${domainInfo.category} domain)`,
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
},
|
},
|
||||||
|
|
|
||||||
|
|
@ -1,41 +1,7 @@
|
||||||
import { PluginOptions, guildPlugin } from "knub";
|
import { guildPlugin } from "knub";
|
||||||
import { hasPhishermanMasterAPIKey, phishermanApiKeyIsValid } from "../../data/Phisherman.js";
|
|
||||||
import { makePublicFn } from "../../pluginUtils.js";
|
|
||||||
import { getDomainInfo } from "./functions/getDomainInfo.js";
|
|
||||||
import { PhishermanPluginType, zPhishermanConfig } from "./types.js";
|
import { PhishermanPluginType, zPhishermanConfig } from "./types.js";
|
||||||
|
|
||||||
export const PhishermanPlugin = guildPlugin<PhishermanPluginType>()({
|
export const PhishermanPlugin = guildPlugin<PhishermanPluginType>()({
|
||||||
name: "phisherman",
|
name: "phisherman",
|
||||||
|
|
||||||
configSchema: zPhishermanConfig,
|
configSchema: zPhishermanConfig,
|
||||||
|
|
||||||
public(pluginData) {
|
|
||||||
return {
|
|
||||||
getDomainInfo: makePublicFn(pluginData, getDomainInfo),
|
|
||||||
};
|
|
||||||
},
|
|
||||||
|
|
||||||
async beforeLoad(pluginData) {
|
|
||||||
const { state } = pluginData;
|
|
||||||
|
|
||||||
pluginData.state.validApiKey = null;
|
|
||||||
|
|
||||||
if (!hasPhishermanMasterAPIKey()) {
|
|
||||||
// tslint:disable-next-line:no-console
|
|
||||||
console.warn("[PHISHERMAN] Could not load Phisherman plugin: master API key is missing");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
const apiKey = pluginData.config.get().api_key;
|
|
||||||
if (apiKey) {
|
|
||||||
const isValid = await phishermanApiKeyIsValid(apiKey).catch((err) => {
|
|
||||||
// tslint:disable-next-line:no-console
|
|
||||||
console.warn(`[PHISHERMAN] Error checking user API key validity:\n${err.toString()}`);
|
|
||||||
return false;
|
|
||||||
});
|
|
||||||
if (isValid) {
|
|
||||||
state.validApiKey = apiKey;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -4,41 +4,12 @@ import { zPhishermanConfig } from "./types.js";
|
||||||
|
|
||||||
export const phishermanPluginDocs: ZeppelinPluginDocs = {
|
export const phishermanPluginDocs: ZeppelinPluginDocs = {
|
||||||
prettyName: "Phisherman",
|
prettyName: "Phisherman",
|
||||||
type: "stable",
|
type: "legacy",
|
||||||
description: trimPluginDescription(`
|
description: trimPluginDescription(`
|
||||||
Match scam/phishing links using the Phisherman API. See https://phisherman.gg/ for more details!
|
Match malicious links using Phisherman
|
||||||
`),
|
`),
|
||||||
configurationGuide: trimPluginDescription(`
|
configurationGuide: trimPluginDescription(`
|
||||||
### Getting started
|
This plugin has been deprecated. Please use the \`include_malicious\` option for automod \`match_links\` trigger instead.
|
||||||
To get started, request an API key for Phisherman following the instructions at https://docs.phisherman.gg/guide/getting-started.html#requesting-api-access
|
|
||||||
Then, add the api key to the plugin's config:
|
|
||||||
|
|
||||||
~~~yml
|
|
||||||
phisherman:
|
|
||||||
config:
|
|
||||||
api_key: "your key here"
|
|
||||||
~~~
|
|
||||||
|
|
||||||
### Note
|
|
||||||
When using Phisherman features in Zeppelin, Zeppelin reports statistics about checked links back to Phisherman. This only includes the domain (e.g. zeppelin.gg), not the full link.
|
|
||||||
|
|
||||||
### Usage with Automod
|
|
||||||
Once you have configured the Phisherman plugin, you are ready to use it with automod. Currently, Phisherman is available as an option in the \`match_links\` plugin:
|
|
||||||
|
|
||||||
~~~yml
|
|
||||||
automod:
|
|
||||||
config:
|
|
||||||
rules:
|
|
||||||
# Clean any scam links detected by Phisherman
|
|
||||||
filter_scam_links:
|
|
||||||
triggers:
|
|
||||||
- match_links:
|
|
||||||
phisherman:
|
|
||||||
include_suspected: true # It's recommended to keep this enabled to catch new scam domains quickly
|
|
||||||
include_verified: true
|
|
||||||
actions:
|
|
||||||
clean: true
|
|
||||||
~~~
|
|
||||||
`),
|
`),
|
||||||
configSchema: zPhishermanConfig,
|
configSchema: zPhishermanConfig,
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -1,31 +0,0 @@
|
||||||
import { GuildPluginData } from "knub";
|
|
||||||
import {
|
|
||||||
getPhishermanDomainInfo,
|
|
||||||
phishermanDomainIsSafe,
|
|
||||||
trackPhishermanCaughtDomain,
|
|
||||||
} from "../../../data/Phisherman.js";
|
|
||||||
import { PhishermanDomainInfo } from "../../../data/types/phisherman.js";
|
|
||||||
import { PhishermanPluginType } from "../types.js";
|
|
||||||
|
|
||||||
export async function getDomainInfo(
|
|
||||||
pluginData: GuildPluginData<PhishermanPluginType>,
|
|
||||||
domain: string,
|
|
||||||
): Promise<PhishermanDomainInfo | null> {
|
|
||||||
if (!pluginData.state.validApiKey) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const info = await getPhishermanDomainInfo(domain).catch((err) => {
|
|
||||||
// tslint:disable-next-line:no-console
|
|
||||||
console.warn(`[PHISHERMAN] Error in getDomainInfo() for server ${pluginData.guild.id}: ${err.message}`);
|
|
||||||
if (err.message === "missing permissions") {
|
|
||||||
pluginData.state.validApiKey = null;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
});
|
|
||||||
if (info != null && !phishermanDomainIsSafe(info)) {
|
|
||||||
trackPhishermanCaughtDomain(pluginData.state.validApiKey, domain);
|
|
||||||
}
|
|
||||||
|
|
||||||
return info;
|
|
||||||
}
|
|
||||||
|
|
@ -7,7 +7,5 @@ export const zPhishermanConfig = z.strictObject({
|
||||||
|
|
||||||
export interface PhishermanPluginType extends BasePluginType {
|
export interface PhishermanPluginType extends BasePluginType {
|
||||||
configSchema: typeof zPhishermanConfig;
|
configSchema: typeof zPhishermanConfig;
|
||||||
state: {
|
state: {};
|
||||||
validApiKey: string | null;
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue