Migrate Censor to new Plugin structure

backend/src/plugins/Censor/CensorPlugin.ts

@@ -0,0 +1,64 @@
+import { zeppelinPlugin } from "../ZeppelinPluginBlueprint";
+import { PluginOptions } from "knub";
+import { ConfigSchema, CensorPluginType } from "./types";
+import { GuildLogs } from "src/data/GuildLogs";
+import { GuildSavedMessages } from "src/data/GuildSavedMessages";
+import { onMessageCreate } from "./util/onMessageCreate";
+import { onMessageUpdate } from "./util/onMessageUpdate";
+
+const defaultOptions: PluginOptions<CensorPluginType> = {
+  config: {
+    filter_zalgo: false,
+    filter_invites: true,
+    invite_guild_whitelist: null,
+    invite_guild_blacklist: null,
+    invite_code_whitelist: null,
+    invite_code_blacklist: null,
+    allow_group_dm_invites: false,
+
+    filter_domains: false,
+    domain_whitelist: null,
+    domain_blacklist: null,
+
+    blocked_tokens: null,
+    blocked_words: null,
+    blocked_regex: null,
+  },
+
+  overrides: [
+    {
+      level: ">=50",
+      config: {
+        filter_zalgo: false,
+        filter_invites: false,
+        filter_domains: false,
+        blocked_tokens: null,
+        blocked_words: null,
+        blocked_regex: null,
+      },
+    },
+  ],
+};
+
+export const CensorPlugin = zeppelinPlugin<CensorPluginType>()("censor", {
+  configSchema: ConfigSchema,
+  defaultOptions,
+
+  onLoad(pluginData) {
+    const { state, guild } = pluginData;
+
+    state.serverLogs = new GuildLogs(guild.id);
+    state.savedMessages = GuildSavedMessages.getGuildInstance(guild.id);
+
+    state.onMessageCreateFn = msg => onMessageCreate(pluginData, msg);
+    state.savedMessages.events.on("create", state.onMessageCreateFn);
+
+    state.onMessageUpdateFn = msg => onMessageUpdate(pluginData, msg);
+    state.savedMessages.events.on("update", state.onMessageUpdateFn);
+  },
+
+  onUnload(pluginData) {
+    pluginData.state.savedMessages.events.off("create", pluginData.state.onMessageCreateFn);
+    pluginData.state.savedMessages.events.off("update", pluginData.state.onMessageUpdateFn);
+  },
+});

backend/src/plugins/Censor/types.ts

@@ -0,0 +1,36 @@
+import * as t from "io-ts";
+import { BasePluginType, eventListener } from "knub";
+import { tNullable } from "src/utils";
+import { TSafeRegex } from "src/validatorUtils";
+import { GuildLogs } from "src/data/GuildLogs";
+import { GuildSavedMessages } from "src/data/GuildSavedMessages";
+
+export const ConfigSchema = t.type({
+  filter_zalgo: t.boolean,
+  filter_invites: t.boolean,
+  invite_guild_whitelist: tNullable(t.array(t.string)),
+  invite_guild_blacklist: tNullable(t.array(t.string)),
+  invite_code_whitelist: tNullable(t.array(t.string)),
+  invite_code_blacklist: tNullable(t.array(t.string)),
+  allow_group_dm_invites: t.boolean,
+  filter_domains: t.boolean,
+  domain_whitelist: tNullable(t.array(t.string)),
+  domain_blacklist: tNullable(t.array(t.string)),
+  blocked_tokens: tNullable(t.array(t.string)),
+  blocked_words: tNullable(t.array(t.string)),
+  blocked_regex: tNullable(t.array(TSafeRegex)),
+});
+export type TConfigSchema = t.TypeOf<typeof ConfigSchema>;
+
+export interface CensorPluginType extends BasePluginType {
+  config: TConfigSchema;
+  state: {
+    serverLogs: GuildLogs;
+    savedMessages: GuildSavedMessages;
+
+    onMessageCreateFn;
+    onMessageUpdateFn;
+  };
+}
+
+export const censorEvent = eventListener<CensorPluginType>();

backend/src/plugins/Censor/util/applyFiltersToMsg.ts

@@ -0,0 +1,157 @@
+import { PluginData } from "knub";
+import { CensorPluginType } from "../types";
+import { SavedMessage } from "src/data/entities/SavedMessage";
+import { Embed, GuildInvite } from "eris";
+import { ZalgoRegex } from "src/data/Zalgo";
+import { getInviteCodesInString, getUrlsInString, resolveMember, resolveInvite } from "src/utils";
+import cloneDeep from "lodash.clonedeep";
+import { censorMessage } from "./censorMessage";
+import escapeStringRegexp from "escape-string-regexp";
+import { logger } from "src/logger";
+
+export async function applyFiltersToMsg(
+  pluginData: PluginData<CensorPluginType>,
+  savedMessage: SavedMessage,
+): Promise<boolean> {
+  const member = await resolveMember(pluginData.client, pluginData.guild, savedMessage.user_id);
+  const config = pluginData.config.getMatchingConfig({ member, channelId: savedMessage.channel_id });
+
+  let messageContent = savedMessage.data.content || "";
+  if (savedMessage.data.attachments) messageContent += " " + JSON.stringify(savedMessage.data.attachments);
+  if (savedMessage.data.embeds) {
+    const embeds = (savedMessage.data.embeds as Embed[]).map(e => cloneDeep(e));
+    for (const embed of embeds) {
+      if (embed.type === "video") {
+        // Ignore video descriptions as they're not actually shown on the embed
+        delete embed.description;
+      }
+    }
+
+    messageContent += " " + JSON.stringify(embeds);
+  }
+
+  // Filter zalgo
+  const filterZalgo = config.filter_zalgo;
+  if (filterZalgo) {
+    const result = ZalgoRegex.exec(messageContent);
+    if (result) {
+      censorMessage(pluginData, savedMessage, "zalgo detected");
+      return true;
+    }
+  }
+
+  // Filter invites
+  const filterInvites = config.filter_invites;
+  if (filterInvites) {
+    const inviteGuildWhitelist = config.invite_guild_whitelist;
+    const inviteGuildBlacklist = config.invite_guild_blacklist;
+    const inviteCodeWhitelist = config.invite_code_whitelist;
+    const inviteCodeBlacklist = config.invite_code_blacklist;
+    const allowGroupDMInvites = config.allow_group_dm_invites;
+
+    const inviteCodes = getInviteCodesInString(messageContent);
+
+    const invites: Array<GuildInvite | null> = await Promise.all(
+      inviteCodes.map(code => resolveInvite(pluginData.client, code)),
+    );
+
+    for (const invite of invites) {
+      // Always filter unknown invites if invite filtering is enabled
+      if (invite == null) {
+        censorMessage(pluginData, savedMessage, `unknown invite not found in whitelist`);
+        return true;
+      }
+
+      if (!invite.guild && !allowGroupDMInvites) {
+        censorMessage(pluginData, savedMessage, `group dm invites are not allowed`);
+        return true;
+      }
+
+      if (inviteGuildWhitelist && !inviteGuildWhitelist.includes(invite.guild.id)) {
+        censorMessage(
+          pluginData,
+          savedMessage,
+          `invite guild (**${invite.guild.name}** \`${invite.guild.id}\`) not found in whitelist`,
+        );
+        return true;
+      }
+
+      if (inviteGuildBlacklist && inviteGuildBlacklist.includes(invite.guild.id)) {
+        censorMessage(
+          pluginData,
+          savedMessage,
+          `invite guild (**${invite.guild.name}** \`${invite.guild.id}\`) found in blacklist`,
+        );
+        return true;
+      }
+
+      if (inviteCodeWhitelist && !inviteCodeWhitelist.includes(invite.code)) {
+        censorMessage(pluginData, savedMessage, `invite code (\`${invite.code}\`) not found in whitelist`);
+        return true;
+      }
+
+      if (inviteCodeBlacklist && inviteCodeBlacklist.includes(invite.code)) {
+        censorMessage(pluginData, savedMessage, `invite code (\`${invite.code}\`) found in blacklist`);
+        return true;
+      }
+    }
+  }
+
+  // Filter domains
+  const filterDomains = config.filter_domains;
+  if (filterDomains) {
+    const domainWhitelist = config.domain_whitelist;
+    const domainBlacklist = config.domain_blacklist;
+
+    const urls = getUrlsInString(messageContent);
+    for (const thisUrl of urls) {
+      if (domainWhitelist && !domainWhitelist.includes(thisUrl.hostname)) {
+        censorMessage(pluginData, savedMessage, `domain (\`${thisUrl.hostname}\`) not found in whitelist`);
+        return true;
+      }
+
+      if (domainBlacklist && domainBlacklist.includes(thisUrl.hostname)) {
+        censorMessage(pluginData, savedMessage, `domain (\`${thisUrl.hostname}\`) found in blacklist`);
+        return true;
+      }
+    }
+  }
+
+  // Filter tokens
+  const blockedTokens = config.blocked_tokens || [];
+  for (const token of blockedTokens) {
+    if (messageContent.toLowerCase().includes(token.toLowerCase())) {
+      censorMessage(pluginData, savedMessage, `blocked token (\`${token}\`) found`);
+      return true;
+    }
+  }
+
+  // Filter words
+  const blockedWords = config.blocked_words || [];
+  for (const word of blockedWords) {
+    const regex = new RegExp(`\\b${escapeStringRegexp(word)}\\b`, "i");
+    if (regex.test(messageContent)) {
+      censorMessage(pluginData, savedMessage, `blocked word (\`${word}\`) found`);
+      return true;
+    }
+  }
+
+  // Filter regex
+  const blockedRegex: RegExp[] = config.blocked_regex || [];
+  for (const [i, regex] of blockedRegex.entries()) {
+    if (typeof regex.test !== "function") {
+      logger.info(
+        `[DEBUG] Regex <${regex}> was not a regex; index ${i} of censor.blocked_regex for guild ${pluginData.guild.name} (${pluginData.guild.id})`,
+      );
+      continue;
+    }
+
+    // We're testing both the original content and content + attachments/embeds here so regexes that use ^ and $ still match the regular content properly
+    if (regex.test(savedMessage.data.content) || regex.test(messageContent)) {
+      censorMessage(pluginData, savedMessage, `blocked regex (\`${regex.source}\`) found`);
+      return true;
+    }
+  }
+
+  return false;
+}

backend/src/plugins/Censor/util/censorMessage.ts

@@ -0,0 +1,31 @@
+import { PluginData } from "knub";
+import { CensorPluginType } from "../types";
+import { SavedMessage } from "src/data/entities/SavedMessage";
+import { LogType } from "src/data/LogType";
+import { stripObjectToScalars, resolveUser } from "src/utils";
+import { disableCodeBlocks, deactivateMentions } from "knub/dist/helpers";
+
+export async function censorMessage(
+  pluginData: PluginData<CensorPluginType>,
+  savedMessage: SavedMessage,
+  reason: string,
+) {
+  pluginData.state.serverLogs.ignoreLog(LogType.MESSAGE_DELETE, savedMessage.id);
+
+  try {
+    await pluginData.client.deleteMessage(savedMessage.channel_id, savedMessage.id, "Censored");
+  } catch (e) {
+    return;
+  }
+
+  const user = await resolveUser(pluginData.client, savedMessage.user_id);
+  const channel = pluginData.guild.channels.get(savedMessage.channel_id);
+
+  pluginData.state.serverLogs.log(LogType.CENSOR, {
+    user: stripObjectToScalars(user),
+    channel: stripObjectToScalars(channel),
+    reason,
+    message: savedMessage,
+    messageText: disableCodeBlocks(deactivateMentions(savedMessage.data.content)),
+  });
+}

backend/src/plugins/Censor/util/onMessageCreate.ts

@@ -0,0 +1,17 @@
+import { PluginData } from "knub";
+import { CensorPluginType } from "../types";
+import { SavedMessage } from "src/data/entities/SavedMessage";
+import { applyFiltersToMsg } from "./applyFiltersToMsg";
+
+export async function onMessageCreate(pluginData: PluginData<CensorPluginType>, savedMessage: SavedMessage) {
+  if (savedMessage.is_bot) return;
+  const lock = await pluginData.locks.acquire(`message-${savedMessage.id}`);
+
+  const wasDeleted = await applyFiltersToMsg(pluginData, savedMessage);
+
+  if (wasDeleted) {
+    lock.interrupt();
+  } else {
+    lock.unlock();
+  }
+}

backend/src/plugins/Censor/util/onMessageUpdate.ts

@@ -0,0 +1,17 @@
+import { PluginData } from "knub";
+import { CensorPluginType } from "../types";
+import { SavedMessage } from "src/data/entities/SavedMessage";
+import { applyFiltersToMsg } from "./applyFiltersToMsg";
+
+export async function onMessageUpdate(pluginData: PluginData<CensorPluginType>, savedMessage: SavedMessage) {
+  if (savedMessage.is_bot) return;
+  const lock = await pluginData.locks.acquire(`message-${savedMessage.id}`);
+
+  const wasDeleted = await applyFiltersToMsg(pluginData, savedMessage);
+
+  if (wasDeleted) {
+    lock.interrupt();
+  } else {
+    lock.unlock();
+  }
+}

backend/src/plugins/availablePlugins.ts

@@ -13,10 +13,12 @@ import { GuildConfigReloaderPlugin } from "./GuildConfigReloader/GuildConfigRelo
 import { CasesPlugin } from "./Cases/CasesPlugin";
 import { MutesPlugin } from "./Mutes/MutesPlugin";
 import { TagsPlugin } from "./Tags/TagsPlugin";
+import { CensorPlugin } from "./Censor/CensorPlugin";
 
 // prettier-ignore
 export const guildPlugins: Array<ZeppelinPluginBlueprint<any>> = [
   AutoReactionsPlugin,
+  CensorPlugin,
   LocateUserPlugin,
   PersistPlugin,
   PingableRolesPlugin,