feat: use webhooks for logs when possible

backend/src/data/Webhooks.ts

@@ -0,0 +1,37 @@
+import { getRepository, Repository } from "typeorm";
+import { Webhook } from "./entities/Webhook";
+import { BaseRepository } from "./BaseRepository";
+import { decrypt, encrypt } from "../utils/crypt";
+
+export class Webhooks extends BaseRepository {
+  repository: Repository<Webhook> = getRepository(Webhook);
+
+  protected async _processEntityFromDB(entity) {
+    entity.token = await decrypt(entity.token);
+    return entity;
+  }
+
+  protected async _processEntityToDB(entity) {
+    entity.token = await encrypt(entity.token);
+    return entity;
+  }
+
+  async findByChannelId(channelId: string): Promise<Webhook | null> {
+    const result = await this.repository.findOne({
+      where: {
+        channel_id: channelId,
+      },
+    });
+
+    return result ? this.processEntityFromDB(result) : null;
+  }
+
+  async create(data: Partial<Webhook>): Promise<void> {
+    data = await this.processEntityToDB(data);
+    await this.repository.insert(data);
+  }
+
+  async delete(id: string): Promise<void> {
+    await this.repository.delete({ id });
+  }
+}

backend/src/data/entities/Webhook.ts

@@ -0,0 +1,14 @@
+import { Column, Entity, PrimaryColumn } from "typeorm";
+
+@Entity("webhooks")
+export class Webhook {
+  @Column()
+  @PrimaryColumn()
+  id: string;
+
+  @Column() guild_id: string;
+
+  @Column() channel_id: string;
+
+  @Column() token: string;
+}

backend/src/migrations/1635779678653-CreateWebhooksTable.ts

@@ -0,0 +1,45 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreateWebhooksTable1635779678653 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: "webhooks",
+        columns: [
+          {
+            name: "id",
+            type: "bigint",
+            isPrimary: true,
+          },
+          {
+            name: "guild_id",
+            type: "bigint",
+          },
+          {
+            name: "channel_id",
+            type: "bigint",
+          },
+          {
+            name: "token",
+            type: "text",
+          },
+          {
+            name: "created_at",
+            type: "datetime",
+            default: "(NOW())",
+          },
+        ],
+        indices: [
+          {
+            columnNames: ["channel_id"],
+            isUnique: true,
+          },
+        ],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropTable("webhooks");
+  }
+}

backend/src/plugins/Cases/CasesPlugin.ts

@@ -16,6 +16,7 @@ import { getRecentCasesByMod } from "./functions/getRecentCasesByMod";
 import { getTotalCasesByMod } from "./functions/getTotalCasesByMod";
 import { postCaseToCaseLogChannel } from "./functions/postToCaseLogChannel";
 import { CaseArgs, CaseNoteArgs, CasesPluginType, ConfigSchema } from "./types";
+import { InternalPosterPlugin } from "../InternalPoster/InternalPosterPlugin";
 
 const defaultOptions = {
   config: {
@@ -40,6 +41,7 @@ export const CasesPlugin = zeppelinGuildPlugin<CasesPluginType>()({
 
   dependencies: async () => [
     TimeAndDatePlugin,
+    InternalPosterPlugin,
     // The `as any` cast here is to prevent TypeScript from locking up from the circular dependency
     ((await import("../Logs/LogsPlugin")) as any).LogsPlugin,
   ],

backend/src/plugins/Cases/functions/postToCaseLogChannel.ts

@@ -1,4 +1,4 @@
-import { FileOptions, Message, MessageOptions, Snowflake, TextChannel } from "discord.js";
+import { FileOptions, Message, MessageOptions, NewsChannel, Snowflake, TextChannel, ThreadChannel } from "discord.js";
 import { GuildPluginData } from "knub";
 import { Case } from "../../../data/entities/Case";
 import { LogType } from "../../../data/LogType";
@@ -7,24 +7,30 @@ import { CasesPluginType } from "../types";
 import { getCaseEmbed } from "./getCaseEmbed";
 import { resolveCaseId } from "./resolveCaseId";
 import { LogsPlugin } from "../../Logs/LogsPlugin";
+import { APIMessage } from "discord-api-types";
+import { InternalPosterPlugin } from "../../InternalPoster/InternalPosterPlugin";
+import { post } from "../../../../../dashboard/src/api";
+import { InternalPosterMessageResult } from "../../InternalPoster/functions/sendMessage";
 
 export async function postToCaseLogChannel(
   pluginData: GuildPluginData<CasesPluginType>,
   content: MessageOptions,
   file?: FileOptions[],
-): Promise<Message | null> {
+): Promise<InternalPosterMessageResult | null> {
   const caseLogChannelId = pluginData.config.get().case_log_channel;
   if (!caseLogChannelId) return null;
 
   const caseLogChannel = pluginData.guild.channels.cache.get(caseLogChannelId as Snowflake);
-  if (!caseLogChannel || !(caseLogChannel instanceof TextChannel)) return null;
+  // This doesn't use `!isText() || isThread()` because TypeScript had some issues inferring types from it
+  if (!caseLogChannel || !(caseLogChannel instanceof TextChannel || caseLogChannel instanceof NewsChannel)) return null;
 
   let result;
   try {
     if (file != null) {
       content.files = file;
     }
-    result = await caseLogChannel.send({ ...content });
+    const poster = pluginData.getPlugin(InternalPosterPlugin);
+    result = await poster.sendMessage(caseLogChannel, { ...content });
   } catch (e) {
     if (isDiscordAPIError(e) && (e.code === 50013 || e.code === 50001)) {
       pluginData.getPlugin(LogsPlugin).logBotAlert({
@@ -42,12 +48,12 @@ export async function postToCaseLogChannel(
 export async function postCaseToCaseLogChannel(
   pluginData: GuildPluginData<CasesPluginType>,
   caseOrCaseId: Case | number,
-): Promise<Message | null> {
+): Promise<void> {
   const theCase = await pluginData.state.cases.find(resolveCaseId(caseOrCaseId));
-  if (!theCase) return null;
+  if (!theCase) return;
 
   const caseEmbed = await getCaseEmbed(pluginData, caseOrCaseId, undefined, true);
-  if (!caseEmbed) return null;
+  if (!caseEmbed) return;
 
   if (theCase.log_message_id) {
     const [channelId, messageId] = theCase.log_message_id.split("-");
@@ -55,7 +61,7 @@ export async function postCaseToCaseLogChannel(
     try {
       const channel = pluginData.guild.channels.resolve(channelId as Snowflake) as TextChannel;
       await channel.messages.edit(messageId as Snowflake, caseEmbed);
-      return null;
+      return;
     } catch {} // tslint:disable-line:no-empty
   }
 
@@ -63,14 +69,13 @@ export async function postCaseToCaseLogChannel(
     const postedMessage = await postToCaseLogChannel(pluginData, caseEmbed);
     if (postedMessage) {
       await pluginData.state.cases.update(theCase.id, {
-        log_message_id: `${postedMessage.channel.id}-${postedMessage.id}`,
+        log_message_id: `${postedMessage.channelId}-${postedMessage.id}`,
       });
     }
-    return postedMessage;
   } catch {
     pluginData.getPlugin(LogsPlugin).logBotAlert({
       body: `Failed to post case #${theCase.case_number} to the case log channel`,
     });
-    return null;
+    return;
   }
 }

backend/src/plugins/InternalPoster/InternalPosterPlugin.ts

@@ -0,0 +1,39 @@
+import { PluginOptions, typedGuildCommand } from "knub";
+import { GuildPingableRoles } from "../../data/GuildPingableRoles";
+import { zeppelinGuildPlugin } from "../ZeppelinPluginBlueprint";
+import { ConfigSchema, InternalPosterPluginType } from "./types";
+import {
+  getPhishermanDomainInfo,
+  hasPhishermanMasterAPIKey,
+  phishermanApiKeyIsValid,
+  reportTrackedDomainsToPhisherman,
+} from "../../data/Phisherman";
+import { mapToPublicFn } from "../../pluginUtils";
+import { Webhooks } from "../../data/Webhooks";
+import { Queue } from "../../Queue";
+import { sendMessage } from "./functions/sendMessage";
+
+const defaultOptions: PluginOptions<InternalPosterPluginType> = {
+  config: {},
+  overrides: [],
+};
+
+export const InternalPosterPlugin = zeppelinGuildPlugin<InternalPosterPluginType>()({
+  name: "internal_poster",
+  showInDocs: false,
+
+  configSchema: ConfigSchema,
+  defaultOptions,
+
+  // prettier-ignore
+  public: {
+    sendMessage: mapToPublicFn(sendMessage),
+  },
+
+  async beforeLoad(pluginData) {
+    pluginData.state.webhooks = new Webhooks();
+    pluginData.state.queue = new Queue();
+    pluginData.state.missingPermissions = false;
+    pluginData.state.webhookClientCache = new Map();
+  },
+});

backend/src/plugins/InternalPoster/functions/getOrCreateWebhookForChannel.ts

@@ -0,0 +1,48 @@
+import { GuildPluginData } from "knub";
+import { InternalPosterPluginType } from "../types";
+import { NewsChannel, Permissions, TextChannel } from "discord.js";
+import { isDiscordAPIError } from "../../../utils";
+
+type WebhookInfo = [id: string, token: string];
+
+export async function getOrCreateWebhookForChannel(
+  pluginData: GuildPluginData<InternalPosterPluginType>,
+  channel: TextChannel | NewsChannel,
+): Promise<WebhookInfo | null> {
+  // tslint:disable-next-line:no-console FIXME: Here for debugging purposes
+  console.log(`getOrCreateWebhookForChannel(${channel.id})`);
+
+  // Database cache
+  const fromDb = await pluginData.state.webhooks.findByChannelId(channel.id);
+  if (fromDb) {
+    return [fromDb.id, fromDb.token];
+  }
+
+  if (pluginData.state.missingPermissions) {
+    return null;
+  }
+
+  // Create new webhook
+  const member = pluginData.client.user && pluginData.guild.members.cache.get(pluginData.client.user.id);
+  if (!member || member.permissions.has(Permissions.FLAGS.MANAGE_WEBHOOKS)) {
+    try {
+      const webhook = await channel.createWebhook(`Zephook ${channel.id}`);
+      await pluginData.state.webhooks.create({
+        id: webhook.id,
+        guild_id: pluginData.guild.id,
+        channel_id: channel.id,
+        token: webhook.token!,
+      });
+      return [webhook.id, webhook.token!];
+    } catch (err) {
+      if (isDiscordAPIError(err) && err.code === 50013) {
+        pluginData.state.missingPermissions = true;
+        console.warn(`Error ${err.code} when trying to create webhook for ${pluginData.guild.id}`);
+        return null;
+      }
+      throw err;
+    }
+  }
+
+  return null;
+}

backend/src/plugins/InternalPoster/functions/sendMessage.ts

@@ -0,0 +1,71 @@
+import { Message, MessageOptions, NewsChannel, TextChannel, WebhookClient } from "discord.js";
+import { GuildPluginData } from "knub";
+import { InternalPosterPluginType } from "../types";
+import { getOrCreateWebhookForChannel } from "./getOrCreateWebhookForChannel";
+import { APIMessage } from "discord-api-types";
+import { isDiscordAPIError } from "../../../utils";
+
+export type InternalPosterMessageResult = {
+  id: string;
+  channelId: string;
+};
+
+/**
+ * Sends a message using a webhook or direct API requests, preferring webhooks when possible.
+ */
+export async function sendMessage(
+  pluginData: GuildPluginData<InternalPosterPluginType>,
+  channel: TextChannel | NewsChannel,
+  content: MessageOptions,
+): Promise<InternalPosterMessageResult> {
+  return pluginData.state.queue.add(async () => {
+    if (!pluginData.state.webhookClientCache.has(channel.id)) {
+      const webhookInfo = await getOrCreateWebhookForChannel(pluginData, channel);
+      if (webhookInfo) {
+        const client = new WebhookClient({
+          id: webhookInfo[0],
+          token: webhookInfo[1],
+        });
+        pluginData.state.webhookClientCache.set(channel.id, client);
+      } else {
+        pluginData.state.webhookClientCache.set(channel.id, null);
+      }
+    }
+
+    const webhookClient = pluginData.state.webhookClientCache.get(channel.id);
+    if (webhookClient) {
+      return webhookClient
+        .send({
+          ...content,
+          ...(pluginData.client.user && {
+            username: pluginData.client.user.username,
+            avatarURL: pluginData.client.user.avatarURL() || pluginData.client.user.defaultAvatarURL,
+          }),
+        })
+        .then((apiMessage) => ({
+          id: apiMessage.id,
+          channelId: apiMessage.channel_id,
+        }))
+        .catch(async (err) => {
+          // Unknown Webhook
+          if (isDiscordAPIError(err) && err.code === 10015) {
+            await pluginData.state.webhooks.delete(webhookClient.id);
+            pluginData.state.webhookClientCache.delete(channel.id);
+
+            // Fallback to regular message for this log message
+            return channel.send(content).then((message) => ({
+              id: message.id,
+              channelId: message.channelId,
+            }));
+          }
+
+          throw err;
+        });
+    }
+
+    return channel.send(content).then((message) => ({
+      id: message.id,
+      channelId: message.channelId,
+    }));
+  });
+}

backend/src/plugins/InternalPoster/types.ts

@@ -0,0 +1,22 @@
+import * as t from "io-ts";
+import { BasePluginType } from "knub";
+import { Webhooks } from "../../data/Webhooks";
+import { Queue } from "../../Queue";
+import { WebhookClient } from "discord.js";
+
+export const ConfigSchema = t.type({});
+export type TConfigSchema = t.TypeOf<typeof ConfigSchema>;
+
+// <channelId, webhookUrl>
+type ChannelWebhookMap = Map<string, string>;
+
+export interface InternalPosterPluginType extends BasePluginType {
+  config: TConfigSchema;
+
+  state: {
+    queue: Queue;
+    webhooks: Webhooks;
+    missingPermissions: boolean;
+    webhookClientCache: Map<string, WebhookClient | null>;
+  };
+}

backend/src/plugins/Logs/LogsPlugin.ts

@@ -112,6 +112,7 @@ import { logVoiceChannelLeave } from "./logFunctions/logVoiceChannelLeave";
 import { logVoiceChannelMove } from "./logFunctions/logVoiceChannelMove";
 import { logMemberTimedUnban } from "./logFunctions/logMemberTimedUnban";
 import { logDmFailed } from "./logFunctions/logDmFailed";
+import { InternalPosterPlugin } from "../InternalPoster/InternalPosterPlugin";
 
 const defaultOptions: PluginOptions<LogsPluginType> = {
   config: {
@@ -145,6 +146,7 @@ export const LogsPlugin = zeppelinGuildPlugin<LogsPluginType>()({
 
   dependencies: async () => [
     TimeAndDatePlugin,
+    InternalPosterPlugin,
     // The `as any` cast here is to prevent TypeScript from locking up from the circular dependency
     ((await import("../Cases/CasesPlugin")) as any).CasesPlugin,
   ],

backend/src/plugins/Logs/util/log.ts

@@ -7,6 +7,7 @@ import { TypedTemplateSafeValueContainer } from "../../../templateFormatter";
 import { LogType } from "../../../data/LogType";
 import { MessageBuffer } from "../../../utils/MessageBuffer";
 import { createChunkedMessage, isDiscordAPIError, MINUTES } from "../../../utils";
+import { InternalPosterPlugin } from "../../InternalPoster/InternalPosterPlugin";
 
 const excludedUserProps = ["user", "member", "mod"];
 const excludedRoleProps = ["message.member.roles", "member.roles"];
@@ -98,6 +99,7 @@ export async function log<TLogType extends keyof ILogTypeData>(
     // Initialize message buffer for this channel
     if (!pluginData.state.buffers.has(channelId)) {
       const batchTime = Math.min(Math.max(opts.batch_time ?? DEFAULT_BATCH_TIME, MIN_BATCH_TIME), MAX_BATCH_TIME);
+      const internalPosterPlugin = pluginData.getPlugin(InternalPosterPlugin);
       pluginData.state.buffers.set(
         channelId,
         new MessageBuffer({
@@ -105,14 +107,12 @@ export async function log<TLogType extends keyof ILogTypeData>(
           textSeparator: "\n",
           consume: (part) => {
             const parse: MessageMentionTypes[] = pluginData.config.get().allow_user_mentions ? ["users"] : [];
-            const promise =
+            internalPosterPlugin
-              part.content && !part.embeds?.length
+              .sendMessage(channel, {
-                ? createChunkedMessage(channel, part.content, { parse })
-                : channel.send({
                 ...part,
                 allowedMentions: { parse },
-                  });
+              })
-            promise.catch((err) => {
+              .catch((err) => {
                 if (isDiscordAPIError(err)) {
                   // Missing Access / Missing Permissions
                   // TODO: Show/log this somewhere
@@ -123,7 +123,9 @@ export async function log<TLogType extends keyof ILogTypeData>(
                 }
 
                 // tslint:disable-next-line:no-console
-              console.warn(`Error while sending ${typeStr} log to ${pluginData.guild.id}/${channelId}: ${err.message}`);
+                console.warn(
+                  `Error while sending ${typeStr} log to ${pluginData.guild.id}/${channelId}: ${err.message}`,
+                );
               });
           },
         }),

backend/src/plugins/availablePlugins.ts

@@ -35,6 +35,7 @@ import { UtilityPlugin } from "./Utility/UtilityPlugin";
 import { WelcomeMessagePlugin } from "./WelcomeMessage/WelcomeMessagePlugin";
 import { ZeppelinGlobalPluginBlueprint, ZeppelinGuildPluginBlueprint } from "./ZeppelinPluginBlueprint";
 import { PhishermanPlugin } from "./Phisherman/PhishermanPlugin";
+import { InternalPosterPlugin } from "./InternalPoster/InternalPosterPlugin";
 
 // prettier-ignore
 export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
@@ -71,6 +72,7 @@ export const guildPlugins: Array<ZeppelinGuildPluginBlueprint<any>> = [
   CountersPlugin,
   ContextMenuPlugin,
   PhishermanPlugin,
+  InternalPosterPlugin,
 ];
 
 // prettier-ignore