Batch embed logs. Always use log batching.

backend/src/plugins/Logs/LogsPlugin.ts

@@ -266,7 +266,7 @@ export const LogsPlugin = zeppelinGuildPlugin<LogsPluginType>()({
     state.archives = GuildArchives.getGuildInstance(guild.id);
     state.cases = GuildCases.getGuildInstance(guild.id);
 
-    state.batches = new Map();
+    state.buffers = new Map();
 
     state.regexRunner = getRegExpRunner(`guild-${pluginData.guild.id}`);
   },

backend/src/plugins/Logs/types.ts

@@ -6,7 +6,7 @@ import { GuildCases } from "../../data/GuildCases";
 import { GuildLogs } from "../../data/GuildLogs";
 import { GuildSavedMessages } from "../../data/GuildSavedMessages";
 import { RegExpRunner } from "../../RegExpRunner";
-import { tMessageContent, tNullable } from "../../utils";
+import { StrictMessageContent, tMessageContent, tNullable } from "../../utils";
 import { TRegex } from "../../validatorUtils";
 import { LogType } from "../../data/LogType";
 import { GuildMember } from "discord.js";
@@ -28,6 +28,7 @@ import {
   TemplateSafeValueContainer,
   TypedTemplateSafeValueContainer,
 } from "../../templateFormatter";
+import { MessageBuffer } from "../../utils/MessageBuffer";
 
 export const tLogFormats = t.record(t.string, t.union([t.string, tMessageContent]));
 export type TLogFormats = t.TypeOf<typeof tLogFormats>;
@@ -85,7 +86,7 @@ export interface LogsPluginType extends BasePluginType {
 
     logListener;
 
-    batches: Map<string, string[]>;
+    buffers: Map<string, MessageBuffer>;
 
     onMessageDeleteFn;
     onMessageDeleteBulkFn;

backend/src/plugins/Logs/util/log.ts

@@ -1,12 +1,11 @@
-import { MessageMentionTypes, Snowflake, TextChannel } from "discord.js";
+import { MessageEmbedOptions, MessageMentionTypes, Snowflake, TextChannel } from "discord.js";
 import { GuildPluginData } from "knub";
-import { SavedMessage } from "../../../data/entities/SavedMessage";
 import { allowTimeout } from "../../../RegExpRunner";
-import { createChunkedMessage, get, noop } from "../../../utils";
+import { ILogTypeData, LogsPluginType, TLogChannel, TLogChannelMap } from "../types";
-import { ILogTypeData, LogsPluginType, LogTypeData, TLogChannelMap } from "../types";
 import { getLogMessage } from "./getLogMessage";
-import { TemplateSafeValueContainer, TypedTemplateSafeValueContainer } from "../../../templateFormatter";
+import { TypedTemplateSafeValueContainer } from "../../../templateFormatter";
 import { LogType } from "../../../data/LogType";
+import { MessageBuffer } from "../../../utils/MessageBuffer";
 
 const excludedUserProps = ["user", "member", "mod"];
 const excludedRoleProps = ["message.member.roles", "member.roles"];
@@ -24,6 +23,53 @@ interface ExclusionData {
   messageTextContent?: string | null;
 }
 
+const DEFAULT_BATCH_TIME = 1000;
+const MIN_BATCH_TIME = 250;
+const MAX_BATCH_TIME = 5000;
+
+async function shouldExclude(
+  pluginData: GuildPluginData<LogsPluginType>,
+  opts: TLogChannel,
+  exclusionData: ExclusionData,
+): Promise<boolean> {
+  if (opts.excluded_users && exclusionData.userId && opts.excluded_users.includes(exclusionData.userId)) {
+    return true;
+  }
+
+  if (opts.exclude_bots && exclusionData.bot) {
+    return true;
+  }
+
+  if (opts.excluded_roles && exclusionData.roles) {
+    for (const role of exclusionData.roles) {
+      if (opts.excluded_roles.includes(role)) {
+        return true;
+      }
+    }
+  }
+
+  if (opts.excluded_channels && exclusionData.channel && opts.excluded_channels.includes(exclusionData.channel)) {
+    return true;
+  }
+
+  if (opts.excluded_categories && exclusionData.category && opts.excluded_categories.includes(exclusionData.category)) {
+    return true;
+  }
+
+  if (opts.excluded_message_regexes && exclusionData.messageTextContent) {
+    for (const regex of opts.excluded_message_regexes) {
+      const matches = await pluginData.state.regexRunner
+        .exec(regex, exclusionData.messageTextContent)
+        .catch(allowTimeout);
+      if (matches) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
 export async function log<TLogType extends keyof ILogTypeData>(
   pluginData: GuildPluginData<LogsPluginType>,
   type: TLogType,
@@ -36,86 +82,47 @@ export async function log<TLogType extends keyof ILogTypeData>(
   logChannelLoop: for (const [channelId, opts] of Object.entries(logChannels)) {
     const channel = pluginData.guild.channels.cache.get(channelId as Snowflake);
     if (!channel || !(channel instanceof TextChannel)) continue;
+    if (opts.include?.length && !opts.include.includes(typeStr)) continue;
+    if (opts.exclude && opts.exclude.includes(typeStr)) continue;
+    if (await shouldExclude(pluginData, opts, exclusionData)) continue;
 
-    if ((opts.include && opts.include.includes(typeStr)) || (opts.exclude && !opts.exclude.includes(typeStr))) {
+    const message = await getLogMessage(pluginData, type, data, {
-      // If this log entry is about an excluded user, skip it
+      format: opts.format,
-      // TODO: Quick and dirty solution, look into changing at some point
+      include_embed_timestamp: opts.include_embed_timestamp,
-      if (opts.excluded_users && exclusionData.userId && opts.excluded_users.includes(exclusionData.userId)) {
+      timestamp_format: opts.timestamp_format,
-        continue;
+    });
-      }
+    if (!message) return;
 
-      // If we're excluding bots and the logged user is a bot, skip it
+    // Initialize message buffer for this channel
-      if (opts.exclude_bots && exclusionData.bot) {
+    if (!pluginData.state.buffers.has(channelId)) {
-        continue;
+      const batchTime = Math.min(Math.max(opts.batch_time ?? DEFAULT_BATCH_TIME, MIN_BATCH_TIME), MAX_BATCH_TIME);
-      }
+      pluginData.state.buffers.set(
-
+        channelId,
-      if (opts.excluded_roles && exclusionData.roles) {
+        new MessageBuffer({
-        for (const role of exclusionData.roles) {
+          timeout: batchTime,
-          if (opts.excluded_roles.includes(role)) {
+          consume: part => {
-            continue logChannelLoop;
+            const parse: MessageMentionTypes[] = pluginData.config.get().allow_user_mentions ? ["users"] : [];
-          }
+            channel
-        }
+              .send({
-      }
+                ...part,
-
+                allowedMentions: { parse },
-      if (opts.excluded_channels && exclusionData.channel && opts.excluded_channels.includes(exclusionData.channel)) {
+              })
-        continue;
+              .catch(err => {
-      }
+                // tslint:disable-next-line:no-console
-
+                console.warn(
-      if (
+                  `Error while sending ${typeStr} log to ${pluginData.guild.id}/${channelId}: ${err.message}`,
-        opts.excluded_categories &&
+                );
-        exclusionData.category &&
+              });
-        opts.excluded_categories.includes(exclusionData.category)
+          },
-      ) {
+        }),
-        continue;
+      );
-      }
-
-      if (opts.excluded_message_regexes && exclusionData.messageTextContent) {
-        for (const regex of opts.excluded_message_regexes) {
-          const matches = await pluginData.state.regexRunner
-            .exec(regex, exclusionData.messageTextContent)
-            .catch(allowTimeout);
-          if (matches) {
-            continue logChannelLoop;
-          }
-        }
-      }
-
-      const message = await getLogMessage(pluginData, type, data, {
-        format: opts.format,
-        include_embed_timestamp: opts.include_embed_timestamp,
-        timestamp_format: opts.timestamp_format,
-      });
-
-      if (message) {
-        // For non-string log messages (i.e. embeds) batching or chunking is not possible, so send them immediately
-        if (typeof message !== "string") {
-          await channel.send(message).catch(noop);
-          return;
-        }
-
-        // Default to batched unless explicitly disabled
-        const batched = opts.batched ?? true;
-        const batchTime = opts.batch_time ?? 1000;
-        const cfg = pluginData.config.get();
-        const parse: MessageMentionTypes[] = cfg.allow_user_mentions ? ["users"] : [];
-
-        if (batched) {
-          // If we're batching log messages, gather all log messages within the set batch_time into a single message
-          if (!pluginData.state.batches.has(channel.id)) {
-            pluginData.state.batches.set(channel.id, []);
-            setTimeout(async () => {
-              const batchedMessage = pluginData.state.batches.get(channel.id)!.join("\n");
-              pluginData.state.batches.delete(channel.id);
-              createChunkedMessage(channel, batchedMessage, { parse }).catch(noop);
-            }, batchTime);
-          }
-
-          pluginData.state.batches.get(channel.id)!.push(message);
-        } else {
-          // If we're not batching log messages, just send them immediately
-          await createChunkedMessage(channel, message, { parse }).catch(noop);
-        }
-      }
     }
+
+    // Add log message to buffer
+    const buffer = pluginData.state.buffers.get(channelId)!;
+    buffer.push({
+      content: typeof message === "string" ? message : message.content || "",
+      embeds: typeof message === "string" ? [] : ((message.embeds || []) as MessageEmbedOptions[]),
+    });
   }
 }

backend/src/utils/MessageBuffer.ts

@@ -0,0 +1,131 @@
+import { StrictMessageContent } from "../utils";
+import Timeout = NodeJS.Timeout;
+
+type ConsumeFn = (part: StrictMessageContent) => void;
+
+type ContentType = "mixed" | "plain" | "embeds";
+
+export type MessageBufferContent = Pick<StrictMessageContent, "content" | "embeds">;
+
+type Chunk = {
+  type: ContentType;
+  content: MessageBufferContent;
+};
+
+export interface MessageBufferOpts {
+  consume?: ConsumeFn;
+  timeout?: number;
+}
+
+const MAX_CHARS_PER_MESSAGE = 2000;
+const MAX_EMBEDS_PER_MESSAGE = 10;
+
+/**
+ * Allows buffering and automatic partitioning of message contents. Useful for e.g. high volume log channels, message chunking, etc.
+ */
+export class MessageBuffer {
+  protected autoConsumeFn: ConsumeFn | null = null;
+
+  protected timeoutMs: number | null = null;
+
+  protected chunk: Chunk | null = null;
+
+  protected chunkTimeout: Timeout | null = null;
+
+  protected finalizedChunks: MessageBufferContent[] = [];
+
+  constructor(opts: MessageBufferOpts = {}) {
+    if (opts.consume) {
+      this.autoConsumeFn = opts.consume;
+    }
+
+    if (opts.timeout) {
+      this.timeoutMs = opts.timeout;
+    }
+  }
+
+  push(content: MessageBufferContent): void {
+    let contentType: ContentType;
+    if (content.content && !content.embeds?.length) {
+      contentType = "plain";
+    } else if (content.embeds?.length && !content.content) {
+      contentType = "embeds";
+    } else {
+      contentType = "mixed";
+    }
+
+    // Plain text can't be merged with mixed or embeds
+    if (contentType === "plain" && this.chunk && this.chunk.type !== "plain") {
+      this.startNewChunk(contentType);
+    }
+    // Mixed can't be merged at all
+    if (contentType === "mixed" && this.chunk) {
+      this.startNewChunk(contentType);
+    }
+
+    if (!this.chunk) this.startNewChunk(contentType);
+    const chunk = this.chunk!;
+
+    if (content.content) {
+      if (chunk.content.content && chunk.content.content.length + content.content.length > MAX_CHARS_PER_MESSAGE) {
+        this.startNewChunk(contentType);
+      }
+
+      if (chunk.content.content == null) chunk.content.content = "";
+      chunk.content.content += content.content;
+    }
+
+    if (content.embeds) {
+      if (chunk.content.embeds && chunk.content.embeds.length + content.embeds.length > MAX_EMBEDS_PER_MESSAGE) {
+        this.startNewChunk(contentType);
+      }
+
+      if (chunk.content.embeds == null) chunk.content.embeds = [];
+      chunk.content.embeds.push(...content.embeds);
+    }
+  }
+
+  protected startNewChunk(type: ContentType): void {
+    if (this.chunk) {
+      this.finalizeChunk();
+    }
+    this.chunk = {
+      type,
+      content: {},
+    };
+    if (this.timeoutMs) {
+      this.chunkTimeout = setTimeout(() => this.finalizeChunk(), this.timeoutMs);
+    }
+  }
+
+  protected finalizeChunk(): void {
+    if (!this.chunk) return;
+    const chunk = this.chunk;
+    this.chunk = null;
+
+    if (this.chunkTimeout) {
+      clearTimeout(this.chunkTimeout);
+      this.chunkTimeout = null;
+    }
+
+    // Discard empty chunks
+    if (!chunk.content.content && !chunk.content.embeds?.length) return;
+
+    if (this.autoConsumeFn) {
+      this.autoConsumeFn(chunk.content);
+      return;
+    }
+
+    this.finalizedChunks.push(chunk.content);
+  }
+
+  consume(): StrictMessageContent[] {
+    return Array.from(this.finalizedChunks);
+    this.finalizedChunks = [];
+  }
+
+  finalizeAndConsume(): StrictMessageContent[] {
+    this.finalizeChunk();
+    return this.consume();
+  }
+}