mirror of
https://github.com/ZeppelinBot/Zeppelin.git
synced 2025-06-17 11:25:03 +00:00
feat: replace Phisherman with FishFish
This commit is contained in:
Dragory
parent
bf08cade15
commit
8c2058821f
13 changed files with 202 additions and 475 deletions
173
backend/src/data/FishFish.ts
Normal file
173
backend/src/data/FishFish.ts
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
import z from "zod/v4";
|
||||
import { env } from "../env.js";
|
||||
import { HOURS, MINUTES, SECONDS } from "../utils.js";
|
||||
|
||||
const API_ROOT = "https://api.fishfish.gg/v1";
|
||||
|
||||
const zDomainCategory = z.literal(["safe", "malware", "phishing"]);
|
||||
|
||||
const zDomain = z.object({
|
||||
name: z.string(),
|
||||
category: zDomainCategory,
|
||||
description: z.string(),
|
||||
added: z.number(),
|
||||
checked: z.number(),
|
||||
});
|
||||
export type FishFishDomain = z.output<typeof zDomain>;
|
||||
|
||||
const FULL_REFRESH_INTERVAL = 6 * HOURS;
|
||||
const domains = new Map<string, FishFishDomain>();
|
||||
|
||||
let sessionTokenPromise: Promise<string> | null = null;
|
||||
|
||||
const WS_RECONNECT_DELAY = 30 * SECONDS;
|
||||
let updatesWs: WebSocket | null = null;
|
||||
|
||||
export class FishFishError extends Error {}
|
||||
|
||||
const zTokenResponse = z.object({
|
||||
expires: z.number(),
|
||||
token: z.string(),
|
||||
});
|
||||
|
||||
async function getSessionToken(): Promise<string> {
|
||||
if (sessionTokenPromise) {
|
||||
return sessionTokenPromise;
|
||||
}
|
||||
|
||||
const apiKey = env.FISHFISH_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new FishFishError("FISHFISH_API_KEY is missing");
|
||||
}
|
||||
|
||||
sessionTokenPromise = (async () => {
|
||||
const response = await fetch(`${API_ROOT}/users/@me/tokens`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: apiKey,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new FishFishError(`Failed to get session token: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const parseResult = zTokenResponse.safeParse(await response.json());
|
||||
if (!parseResult.success) {
|
||||
throw new FishFishError(`Parse error when fetching session token: ${parseResult.error.message}`);
|
||||
}
|
||||
|
||||
const timeUntilExpiry = Date.now() - parseResult.data.expires * 1000;
|
||||
setTimeout(() => {
|
||||
sessionTokenPromise = null;
|
||||
}, timeUntilExpiry - 1 * MINUTES); // Subtract a minute to ensure we refresh before expiry
|
||||
|
||||
return parseResult.data.token;
|
||||
})();
|
||||
sessionTokenPromise.catch((err) => {
|
||||
sessionTokenPromise = null;
|
||||
throw err;
|
||||
});
|
||||
|
||||
return sessionTokenPromise;
|
||||
}
|
||||
|
||||
async function fishFishApiCall(method: string, path: string, query: Record<string, string> = {}): Promise<unknown> {
|
||||
const sessionToken = await getSessionToken();
|
||||
const queryParams = new URLSearchParams(query);
|
||||
const response = await fetch(`https://api.fishfish.gg/v1/${path}?${queryParams}`, {
|
||||
method,
|
||||
headers: {
|
||||
Authorization: sessionToken,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new FishFishError(`FishFish API call failed: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function subscribeToFishFishUpdates(): Promise<void> {
|
||||
if (updatesWs) {
|
||||
return;
|
||||
}
|
||||
const sessionToken = await getSessionToken();
|
||||
console.log("[FISHFISH] Connecting to WebSocket for real-time updates");
|
||||
updatesWs = new WebSocket("wss://api.fishfish.gg/v1/stream", {
|
||||
headers: {
|
||||
Authorization: sessionToken,
|
||||
},
|
||||
});
|
||||
updatesWs.addEventListener("open", () => {
|
||||
console.log("[FISHFISH] WebSocket connection established");
|
||||
});
|
||||
updatesWs.addEventListener("message", (event) => {
|
||||
console.log("[FISHFISH] ws update:", event.data);
|
||||
});
|
||||
updatesWs.addEventListener("error", (error) => {
|
||||
console.error(`[FISHFISH] WebSocket error: ${error.message}`);
|
||||
});
|
||||
updatesWs.addEventListener("close", () => {
|
||||
console.log("[FISHFISH] WebSocket connection closed, reconnecting after delay");
|
||||
updatesWs = null;
|
||||
setTimeout(() => {
|
||||
subscribeToFishFishUpdates();
|
||||
}, WS_RECONNECT_DELAY);
|
||||
});
|
||||
}
|
||||
|
||||
async function refreshFishFishDomains() {
|
||||
const rawData = await fishFishApiCall("GET", "domains", { full: "true" });
|
||||
const parseResult = z.array(zDomain).safeParse(rawData);
|
||||
if (!parseResult.success) {
|
||||
throw new FishFishError(`Parse error when refreshing domains: ${parseResult.error.message}`);
|
||||
}
|
||||
|
||||
domains.clear();
|
||||
for (const domain of parseResult.data) {
|
||||
domains.set(domain.name, domain);
|
||||
}
|
||||
|
||||
domains.set("malware-link.test.zeppelin.gg", {
|
||||
name: "malware-link.test.zeppelin.gg",
|
||||
category: "malware",
|
||||
description: "",
|
||||
added: Date.now(),
|
||||
checked: Date.now(),
|
||||
});
|
||||
domains.set("phishing-link.test.zeppelin.gg", {
|
||||
name: "phishing-link.test.zeppelin.gg",
|
||||
category: "phishing",
|
||||
description: "",
|
||||
added: Date.now(),
|
||||
checked: Date.now(),
|
||||
});
|
||||
domains.set("safe-link.test.zeppelin.gg", {
|
||||
name: "safe-link.test.zeppelin.gg",
|
||||
category: "safe",
|
||||
description: "",
|
||||
added: Date.now(),
|
||||
checked: Date.now(),
|
||||
});
|
||||
|
||||
console.log("[FISHFISH] Refreshed FishFish domains, total count:", domains.size);
|
||||
}
|
||||
|
||||
export async function initFishFish() {
|
||||
if (!env.FISHFISH_API_KEY) {
|
||||
console.warn("[FISHFISH] FISHFISH_API_KEY is not set, FishFish functionality will be disabled.");
|
||||
return;
|
||||
}
|
||||
|
||||
await refreshFishFishDomains();
|
||||
void subscribeToFishFishUpdates();
|
||||
setInterval(() => refreshFishFishDomains(), FULL_REFRESH_INTERVAL);
|
||||
}
|
||||
|
||||
export function getFishFishDomain(domain: string): FishFishDomain | undefined {
|
||||
return domains.get(domain.toLowerCase());
|
||||
}
|
||||
|
|
@ -1,253 +0,0 @@
|
|||
import crypto from "crypto";
|
||||
import moment from "moment-timezone";
|
||||
import { Repository } from "typeorm";
|
||||
import { env } from "../env.js";
|
||||
import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils.js";
|
||||
import { dataSource } from "./dataSource.js";
|
||||
import { PhishermanCacheEntry } from "./entities/PhishermanCacheEntry.js";
|
||||
import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry.js";
|
||||
import { PhishermanDomainInfo, PhishermanUnknownDomain } from "./types/phisherman.js";
|
||||
|
||||
const API_URL = "https://api.phisherman.gg";
|
||||
const MASTER_API_KEY = env.PHISHERMAN_API_KEY;
|
||||
|
||||
let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
|
||||
|
||||
const pendingApiRequests: Map<string, Promise<unknown>> = new Map();
|
||||
const pendingDomainInfoChecks: Map<string, Promise<PhishermanDomainInfo | null>> = new Map();
|
||||
|
||||
type MemoryCacheEntry = {
|
||||
info: PhishermanDomainInfo | null;
|
||||
expires: number;
|
||||
};
|
||||
const memoryCache: Map<string, MemoryCacheEntry> = new Map();
|
||||
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, entry] of memoryCache.entries()) {
|
||||
if (entry.expires <= now) {
|
||||
memoryCache.delete(key);
|
||||
}
|
||||
}
|
||||
}, 2 * MINUTES);
|
||||
|
||||
const UNKNOWN_DOMAIN_CACHE_LIFETIME = 2 * MINUTES;
|
||||
const DETECTED_DOMAIN_CACHE_LIFETIME = 15 * MINUTES;
|
||||
const SAFE_DOMAIN_CACHE_LIFETIME = 7 * DAYS;
|
||||
|
||||
const KEY_VALIDITY_LIFETIME = 24 * HOURS;
|
||||
|
||||
let cacheRepository: Repository<PhishermanCacheEntry> | null = null;
|
||||
function getCacheRepository(): Repository<PhishermanCacheEntry> {
|
||||
if (cacheRepository == null) {
|
||||
cacheRepository = dataSource.getRepository(PhishermanCacheEntry);
|
||||
}
|
||||
return cacheRepository;
|
||||
}
|
||||
|
||||
let keyCacheRepository: Repository<PhishermanKeyCacheEntry> | null = null;
|
||||
function getKeyCacheRepository(): Repository<PhishermanKeyCacheEntry> {
|
||||
if (keyCacheRepository == null) {
|
||||
keyCacheRepository = dataSource.getRepository(PhishermanKeyCacheEntry);
|
||||
}
|
||||
return keyCacheRepository;
|
||||
}
|
||||
|
||||
class PhishermanApiError extends Error {
|
||||
method: string;
|
||||
url: string;
|
||||
status: number;
|
||||
|
||||
constructor(method: string, url: string, status: number, message: string) {
|
||||
super(message);
|
||||
this.method = method;
|
||||
this.url = url;
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
toString() {
|
||||
return `Error ${this.status} in ${this.method} ${this.url}: ${this.message}`;
|
||||
}
|
||||
}
|
||||
|
||||
export function hasPhishermanMasterAPIKey() {
|
||||
return MASTER_API_KEY != null && MASTER_API_KEY !== "";
|
||||
}
|
||||
|
||||
export function phishermanDomainIsSafe(info: PhishermanDomainInfo): boolean {
|
||||
return info.classification === "safe";
|
||||
}
|
||||
|
||||
const leadingSlashRegex = /^\/+/g;
|
||||
function trimLeadingSlash(str: string): string {
|
||||
return str.replace(leadingSlashRegex, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Make an arbitrary API call to the Phisherman API
|
||||
*/
|
||||
async function apiCall<T>(
|
||||
method: "GET" | "POST",
|
||||
resource: string,
|
||||
payload?: Record<string, unknown> | null,
|
||||
): Promise<T> {
|
||||
if (!hasPhishermanMasterAPIKey()) {
|
||||
throw new Error("Phisherman master API key missing");
|
||||
}
|
||||
|
||||
const url = `${API_URL}/${trimLeadingSlash(resource)}`;
|
||||
const key = `${method} ${url}`;
|
||||
|
||||
if (pendingApiRequests.has(key)) {
|
||||
return pendingApiRequests.get(key)! as Promise<T>;
|
||||
}
|
||||
|
||||
let requestPromise = (async () => {
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
headers: new Headers({
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${MASTER_API_KEY}`,
|
||||
}),
|
||||
body: payload ? JSON.stringify(payload) : undefined,
|
||||
});
|
||||
const data = await response.json().catch(() => null);
|
||||
if (!response.ok || (data as any)?.success === false) {
|
||||
throw new PhishermanApiError(method, url, response.status, (data as any)?.message ?? "");
|
||||
}
|
||||
return data;
|
||||
})();
|
||||
requestPromise = requestPromise.finally(() => {
|
||||
pendingApiRequests.delete(key);
|
||||
});
|
||||
pendingApiRequests.set(key, requestPromise);
|
||||
return requestPromise as Promise<T>;
|
||||
}
|
||||
|
||||
type DomainInfoApiCallResult = PhishermanUnknownDomain | PhishermanDomainInfo;
|
||||
async function fetchDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
||||
// tslint:disable-next-line:no-console
|
||||
console.log(`[PHISHERMAN] Requesting domain information: ${domain}`);
|
||||
const result = await apiCall<Record<string, DomainInfoApiCallResult>>("GET", `/v2/domains/info/${domain}`);
|
||||
const firstKey = Object.keys(result)[0];
|
||||
const domainInfo = firstKey ? result[firstKey] : null;
|
||||
if (!domainInfo) {
|
||||
// tslint:disable-next-line:no-console
|
||||
console.warn(`Unexpected Phisherman API response for ${domain}:`, result);
|
||||
return null;
|
||||
}
|
||||
if (domainInfo.classification === "unknown") {
|
||||
return null;
|
||||
}
|
||||
return domainInfo;
|
||||
}
|
||||
|
||||
export async function getPhishermanDomainInfo(domain: string): Promise<PhishermanDomainInfo | null> {
|
||||
if (pendingDomainInfoChecks.has(domain)) {
|
||||
return pendingDomainInfoChecks.get(domain)!;
|
||||
}
|
||||
|
||||
let promise = (async () => {
|
||||
if (memoryCache.has(domain)) {
|
||||
return memoryCache.get(domain)!.info;
|
||||
}
|
||||
|
||||
const dbCache = getCacheRepository();
|
||||
const existingCachedEntry = await dbCache.findOne({
|
||||
where: { domain },
|
||||
});
|
||||
if (existingCachedEntry) {
|
||||
return existingCachedEntry.data;
|
||||
}
|
||||
|
||||
const freshData = await fetchDomainInfo(domain);
|
||||
const expiryTime =
|
||||
freshData === null
|
||||
? UNKNOWN_DOMAIN_CACHE_LIFETIME
|
||||
: phishermanDomainIsSafe(freshData)
|
||||
? SAFE_DOMAIN_CACHE_LIFETIME
|
||||
: DETECTED_DOMAIN_CACHE_LIFETIME;
|
||||
memoryCache.set(domain, {
|
||||
info: freshData,
|
||||
expires: Date.now() + expiryTime,
|
||||
});
|
||||
|
||||
if (freshData) {
|
||||
// Database cache only stores safe/detected domains, not unknown ones
|
||||
await dbCache.insert({
|
||||
domain,
|
||||
data: freshData,
|
||||
expires_at: moment().add(expiryTime, "ms").format(DBDateFormat),
|
||||
});
|
||||
}
|
||||
|
||||
return freshData;
|
||||
})();
|
||||
promise = promise.finally(() => {
|
||||
pendingDomainInfoChecks.delete(domain);
|
||||
});
|
||||
pendingDomainInfoChecks.set(domain, promise);
|
||||
|
||||
return promise;
|
||||
}
|
||||
|
||||
export async function phishermanApiKeyIsValid(apiKey: string): Promise<boolean> {
|
||||
if (apiKey === MASTER_API_KEY) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const keyCache = getKeyCacheRepository();
|
||||
const hash = crypto.createHash("sha256").update(apiKey).digest("hex");
|
||||
const entry = await keyCache.findOne({
|
||||
where: { hash },
|
||||
});
|
||||
if (entry) {
|
||||
return entry.is_valid;
|
||||
}
|
||||
|
||||
const { valid: isValid } = await apiCall<{ valid: boolean }>("POST", "/zeppelin/check-key", { apiKey });
|
||||
|
||||
await keyCache.insert({
|
||||
hash,
|
||||
is_valid: isValid,
|
||||
expires_at: moment().add(KEY_VALIDITY_LIFETIME, "ms").format(DBDateFormat),
|
||||
});
|
||||
|
||||
return isValid;
|
||||
}
|
||||
|
||||
export function trackPhishermanCaughtDomain(apiKey: string, domain: string) {
|
||||
if (!caughtDomainTrackingMap.has(apiKey)) {
|
||||
caughtDomainTrackingMap.set(apiKey, new Map());
|
||||
}
|
||||
const apiKeyMap = caughtDomainTrackingMap.get(apiKey)!;
|
||||
if (!apiKeyMap.has(domain)) {
|
||||
apiKeyMap.set(domain, []);
|
||||
}
|
||||
const timestamps = apiKeyMap.get(domain)!;
|
||||
timestamps.push(Date.now());
|
||||
}
|
||||
|
||||
export async function reportTrackedDomainsToPhisherman() {
|
||||
const result = {};
|
||||
for (const [apiKey, domains] of caughtDomainTrackingMap.entries()) {
|
||||
result[apiKey] = {};
|
||||
for (const [domain, timestamps] of domains.entries()) {
|
||||
result[apiKey][domain] = timestamps;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(result).length > 0) {
|
||||
await apiCall("POST", "/v2/phish/caught/bulk", result);
|
||||
caughtDomainTrackingMap = new Map();
|
||||
}
|
||||
}
|
||||
|
||||
export async function deleteStalePhishermanCacheEntries() {
|
||||
await getCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
||||
}
|
||||
|
||||
export async function deleteStalePhishermanKeyCacheEntries() {
|
||||
await getKeyCacheRepository().createQueryBuilder().where("expires_at <= NOW()").delete().execute();
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
import { Column, Entity, PrimaryColumn } from "typeorm";
|
||||
import { PhishermanDomainInfo } from "../types/phisherman.js";
|
||||
|
||||
@Entity("phisherman_cache")
|
||||
export class PhishermanCacheEntry {
|
||||
@Column()
|
||||
@PrimaryColumn()
|
||||
id: number;
|
||||
|
||||
@Column()
|
||||
domain: string;
|
||||
|
||||
@Column("simple-json")
|
||||
data: PhishermanDomainInfo;
|
||||
|
||||
@Column()
|
||||
expires_at: string;
|
||||
}
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
import { Column, Entity, PrimaryColumn } from "typeorm";
|
||||
|
||||
@Entity("phisherman_key_cache")
|
||||
export class PhishermanKeyCacheEntry {
|
||||
@Column()
|
||||
@PrimaryColumn()
|
||||
id: number;
|
||||
|
||||
@Column()
|
||||
hash: string;
|
||||
|
||||
@Column()
|
||||
is_valid: boolean;
|
||||
|
||||
@Column()
|
||||
expires_at: string;
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
// tslint:disable:no-console
|
||||
|
||||
import { MINUTES } from "../../utils.js";
|
||||
import {
|
||||
deleteStalePhishermanCacheEntries,
|
||||
deleteStalePhishermanKeyCacheEntries,
|
||||
reportTrackedDomainsToPhisherman,
|
||||
} from "../Phisherman.js";
|
||||
|
||||
const CACHE_CLEANUP_LOOP_INTERVAL = 15 * MINUTES;
|
||||
const REPORT_LOOP_INTERVAL = 15 * MINUTES;
|
||||
|
||||
export async function runPhishermanCacheCleanupLoop() {
|
||||
console.log("[PHISHERMAN] Deleting stale cache entries");
|
||||
await deleteStalePhishermanCacheEntries().catch((err) => console.warn(err));
|
||||
|
||||
console.log("[PHISHERMAN] Deleting stale key cache entries");
|
||||
await deleteStalePhishermanKeyCacheEntries().catch((err) => console.warn(err));
|
||||
|
||||
setTimeout(() => runPhishermanCacheCleanupLoop(), CACHE_CLEANUP_LOOP_INTERVAL);
|
||||
}
|
||||
|
||||
export async function runPhishermanReportingLoop() {
|
||||
console.log("[PHISHERMAN] Reporting tracked domains");
|
||||
await reportTrackedDomainsToPhisherman().catch((err) => console.warn(err));
|
||||
|
||||
setTimeout(() => runPhishermanReportingLoop(), REPORT_LOOP_INTERVAL);
|
||||
}
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
export interface PhishermanUnknownDomain {
|
||||
classification: "unknown";
|
||||
}
|
||||
|
||||
export interface PhishermanDomainInfo {
|
||||
status: string;
|
||||
lastChecked: string;
|
||||
verifiedPhish: boolean;
|
||||
classification: "safe" | "malicious";
|
||||
created: string;
|
||||
firstSeen: string | null;
|
||||
lastSeen: string | null;
|
||||
targetedBrand: string;
|
||||
phishCaught: number;
|
||||
details: PhishermanDomainInfoDetails;
|
||||
}
|
||||
|
||||
export interface PhishermanDomainInfoDetails {
|
||||
phishTankId: string | null;
|
||||
urlScanId: string;
|
||||
websiteScreenshot: string;
|
||||
ip_address: string;
|
||||
asn: PhishermanDomainInfoAsn;
|
||||
registry: string;
|
||||
country: string;
|
||||
}
|
||||
|
||||
export interface PhishermanDomainInfoAsn {
|
||||
asn: string;
|
||||
asn_name: string;
|
||||
route: string;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue