From 001b6d00ea4fb86e2d5cbae45005a40f2c7bafbc Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sat, 11 Sep 2021 18:57:06 +0300 Subject: [PATCH] Batch embed logs. Always use log batching. --- backend/src/plugins/Logs/LogsPlugin.ts | 2 +- backend/src/plugins/Logs/types.ts | 5 +- backend/src/plugins/Logs/util/log.ts | 173 +++++++++++++------------ backend/src/utils/MessageBuffer.ts | 131 +++++++++++++++++++ 4 files changed, 225 insertions(+), 86 deletions(-) create mode 100644 backend/src/utils/MessageBuffer.ts diff --git a/backend/src/plugins/Logs/LogsPlugin.ts b/backend/src/plugins/Logs/LogsPlugin.ts index 03423404..a6eb8443 100644 --- a/backend/src/plugins/Logs/LogsPlugin.ts +++ b/backend/src/plugins/Logs/LogsPlugin.ts @@ -266,7 +266,7 @@ export const LogsPlugin = zeppelinGuildPlugin()({ state.archives = GuildArchives.getGuildInstance(guild.id); state.cases = GuildCases.getGuildInstance(guild.id); - state.batches = new Map(); + state.buffers = new Map(); state.regexRunner = getRegExpRunner(`guild-${pluginData.guild.id}`); }, diff --git a/backend/src/plugins/Logs/types.ts b/backend/src/plugins/Logs/types.ts index c952349d..39d9d440 100644 --- a/backend/src/plugins/Logs/types.ts +++ b/backend/src/plugins/Logs/types.ts @@ -6,7 +6,7 @@ import { GuildCases } from "../../data/GuildCases"; import { GuildLogs } from "../../data/GuildLogs"; import { GuildSavedMessages } from "../../data/GuildSavedMessages"; import { RegExpRunner } from "../../RegExpRunner"; -import { tMessageContent, tNullable } from "../../utils"; +import { StrictMessageContent, tMessageContent, tNullable } from "../../utils"; import { TRegex } from "../../validatorUtils"; import { LogType } from "../../data/LogType"; import { GuildMember } from "discord.js"; @@ -28,6 +28,7 @@ import { TemplateSafeValueContainer, TypedTemplateSafeValueContainer, } from "../../templateFormatter"; +import { MessageBuffer } from "../../utils/MessageBuffer"; export const tLogFormats = t.record(t.string, t.union([t.string, tMessageContent])); export type TLogFormats = t.TypeOf; @@ -85,7 +86,7 @@ export interface LogsPluginType extends BasePluginType { logListener; - batches: Map; + buffers: Map; onMessageDeleteFn; onMessageDeleteBulkFn; diff --git a/backend/src/plugins/Logs/util/log.ts b/backend/src/plugins/Logs/util/log.ts index 45205882..0afc1e1d 100644 --- a/backend/src/plugins/Logs/util/log.ts +++ b/backend/src/plugins/Logs/util/log.ts @@ -1,12 +1,11 @@ -import { MessageMentionTypes, Snowflake, TextChannel } from "discord.js"; +import { MessageEmbedOptions, MessageMentionTypes, Snowflake, TextChannel } from "discord.js"; import { GuildPluginData } from "knub"; -import { SavedMessage } from "../../../data/entities/SavedMessage"; import { allowTimeout } from "../../../RegExpRunner"; -import { createChunkedMessage, get, noop } from "../../../utils"; -import { ILogTypeData, LogsPluginType, LogTypeData, TLogChannelMap } from "../types"; +import { ILogTypeData, LogsPluginType, TLogChannel, TLogChannelMap } from "../types"; import { getLogMessage } from "./getLogMessage"; -import { TemplateSafeValueContainer, TypedTemplateSafeValueContainer } from "../../../templateFormatter"; +import { TypedTemplateSafeValueContainer } from "../../../templateFormatter"; import { LogType } from "../../../data/LogType"; +import { MessageBuffer } from "../../../utils/MessageBuffer"; const excludedUserProps = ["user", "member", "mod"]; const excludedRoleProps = ["message.member.roles", "member.roles"]; @@ -24,6 +23,53 @@ interface ExclusionData { messageTextContent?: string | null; } +const DEFAULT_BATCH_TIME = 1000; +const MIN_BATCH_TIME = 250; +const MAX_BATCH_TIME = 5000; + +async function shouldExclude( + pluginData: GuildPluginData, + opts: TLogChannel, + exclusionData: ExclusionData, +): Promise { + if (opts.excluded_users && exclusionData.userId && opts.excluded_users.includes(exclusionData.userId)) { + return true; + } + + if (opts.exclude_bots && exclusionData.bot) { + return true; + } + + if (opts.excluded_roles && exclusionData.roles) { + for (const role of exclusionData.roles) { + if (opts.excluded_roles.includes(role)) { + return true; + } + } + } + + if (opts.excluded_channels && exclusionData.channel && opts.excluded_channels.includes(exclusionData.channel)) { + return true; + } + + if (opts.excluded_categories && exclusionData.category && opts.excluded_categories.includes(exclusionData.category)) { + return true; + } + + if (opts.excluded_message_regexes && exclusionData.messageTextContent) { + for (const regex of opts.excluded_message_regexes) { + const matches = await pluginData.state.regexRunner + .exec(regex, exclusionData.messageTextContent) + .catch(allowTimeout); + if (matches) { + return true; + } + } + } + + return false; +} + export async function log( pluginData: GuildPluginData, type: TLogType, @@ -36,86 +82,47 @@ export async function log( logChannelLoop: for (const [channelId, opts] of Object.entries(logChannels)) { const channel = pluginData.guild.channels.cache.get(channelId as Snowflake); if (!channel || !(channel instanceof TextChannel)) continue; + if (opts.include?.length && !opts.include.includes(typeStr)) continue; + if (opts.exclude && opts.exclude.includes(typeStr)) continue; + if (await shouldExclude(pluginData, opts, exclusionData)) continue; - if ((opts.include && opts.include.includes(typeStr)) || (opts.exclude && !opts.exclude.includes(typeStr))) { - // If this log entry is about an excluded user, skip it - // TODO: Quick and dirty solution, look into changing at some point - if (opts.excluded_users && exclusionData.userId && opts.excluded_users.includes(exclusionData.userId)) { - continue; - } + const message = await getLogMessage(pluginData, type, data, { + format: opts.format, + include_embed_timestamp: opts.include_embed_timestamp, + timestamp_format: opts.timestamp_format, + }); + if (!message) return; - // If we're excluding bots and the logged user is a bot, skip it - if (opts.exclude_bots && exclusionData.bot) { - continue; - } - - if (opts.excluded_roles && exclusionData.roles) { - for (const role of exclusionData.roles) { - if (opts.excluded_roles.includes(role)) { - continue logChannelLoop; - } - } - } - - if (opts.excluded_channels && exclusionData.channel && opts.excluded_channels.includes(exclusionData.channel)) { - continue; - } - - if ( - opts.excluded_categories && - exclusionData.category && - opts.excluded_categories.includes(exclusionData.category) - ) { - continue; - } - - if (opts.excluded_message_regexes && exclusionData.messageTextContent) { - for (const regex of opts.excluded_message_regexes) { - const matches = await pluginData.state.regexRunner - .exec(regex, exclusionData.messageTextContent) - .catch(allowTimeout); - if (matches) { - continue logChannelLoop; - } - } - } - - const message = await getLogMessage(pluginData, type, data, { - format: opts.format, - include_embed_timestamp: opts.include_embed_timestamp, - timestamp_format: opts.timestamp_format, - }); - - if (message) { - // For non-string log messages (i.e. embeds) batching or chunking is not possible, so send them immediately - if (typeof message !== "string") { - await channel.send(message).catch(noop); - return; - } - - // Default to batched unless explicitly disabled - const batched = opts.batched ?? true; - const batchTime = opts.batch_time ?? 1000; - const cfg = pluginData.config.get(); - const parse: MessageMentionTypes[] = cfg.allow_user_mentions ? ["users"] : []; - - if (batched) { - // If we're batching log messages, gather all log messages within the set batch_time into a single message - if (!pluginData.state.batches.has(channel.id)) { - pluginData.state.batches.set(channel.id, []); - setTimeout(async () => { - const batchedMessage = pluginData.state.batches.get(channel.id)!.join("\n"); - pluginData.state.batches.delete(channel.id); - createChunkedMessage(channel, batchedMessage, { parse }).catch(noop); - }, batchTime); - } - - pluginData.state.batches.get(channel.id)!.push(message); - } else { - // If we're not batching log messages, just send them immediately - await createChunkedMessage(channel, message, { parse }).catch(noop); - } - } + // Initialize message buffer for this channel + if (!pluginData.state.buffers.has(channelId)) { + const batchTime = Math.min(Math.max(opts.batch_time ?? DEFAULT_BATCH_TIME, MIN_BATCH_TIME), MAX_BATCH_TIME); + pluginData.state.buffers.set( + channelId, + new MessageBuffer({ + timeout: batchTime, + consume: part => { + const parse: MessageMentionTypes[] = pluginData.config.get().allow_user_mentions ? ["users"] : []; + channel + .send({ + ...part, + allowedMentions: { parse }, + }) + .catch(err => { + // tslint:disable-next-line:no-console + console.warn( + `Error while sending ${typeStr} log to ${pluginData.guild.id}/${channelId}: ${err.message}`, + ); + }); + }, + }), + ); } + + // Add log message to buffer + const buffer = pluginData.state.buffers.get(channelId)!; + buffer.push({ + content: typeof message === "string" ? message : message.content || "", + embeds: typeof message === "string" ? [] : ((message.embeds || []) as MessageEmbedOptions[]), + }); } } diff --git a/backend/src/utils/MessageBuffer.ts b/backend/src/utils/MessageBuffer.ts new file mode 100644 index 00000000..06000d87 --- /dev/null +++ b/backend/src/utils/MessageBuffer.ts @@ -0,0 +1,131 @@ +import { StrictMessageContent } from "../utils"; +import Timeout = NodeJS.Timeout; + +type ConsumeFn = (part: StrictMessageContent) => void; + +type ContentType = "mixed" | "plain" | "embeds"; + +export type MessageBufferContent = Pick; + +type Chunk = { + type: ContentType; + content: MessageBufferContent; +}; + +export interface MessageBufferOpts { + consume?: ConsumeFn; + timeout?: number; +} + +const MAX_CHARS_PER_MESSAGE = 2000; +const MAX_EMBEDS_PER_MESSAGE = 10; + +/** + * Allows buffering and automatic partitioning of message contents. Useful for e.g. high volume log channels, message chunking, etc. + */ +export class MessageBuffer { + protected autoConsumeFn: ConsumeFn | null = null; + + protected timeoutMs: number | null = null; + + protected chunk: Chunk | null = null; + + protected chunkTimeout: Timeout | null = null; + + protected finalizedChunks: MessageBufferContent[] = []; + + constructor(opts: MessageBufferOpts = {}) { + if (opts.consume) { + this.autoConsumeFn = opts.consume; + } + + if (opts.timeout) { + this.timeoutMs = opts.timeout; + } + } + + push(content: MessageBufferContent): void { + let contentType: ContentType; + if (content.content && !content.embeds?.length) { + contentType = "plain"; + } else if (content.embeds?.length && !content.content) { + contentType = "embeds"; + } else { + contentType = "mixed"; + } + + // Plain text can't be merged with mixed or embeds + if (contentType === "plain" && this.chunk && this.chunk.type !== "plain") { + this.startNewChunk(contentType); + } + // Mixed can't be merged at all + if (contentType === "mixed" && this.chunk) { + this.startNewChunk(contentType); + } + + if (!this.chunk) this.startNewChunk(contentType); + const chunk = this.chunk!; + + if (content.content) { + if (chunk.content.content && chunk.content.content.length + content.content.length > MAX_CHARS_PER_MESSAGE) { + this.startNewChunk(contentType); + } + + if (chunk.content.content == null) chunk.content.content = ""; + chunk.content.content += content.content; + } + + if (content.embeds) { + if (chunk.content.embeds && chunk.content.embeds.length + content.embeds.length > MAX_EMBEDS_PER_MESSAGE) { + this.startNewChunk(contentType); + } + + if (chunk.content.embeds == null) chunk.content.embeds = []; + chunk.content.embeds.push(...content.embeds); + } + } + + protected startNewChunk(type: ContentType): void { + if (this.chunk) { + this.finalizeChunk(); + } + this.chunk = { + type, + content: {}, + }; + if (this.timeoutMs) { + this.chunkTimeout = setTimeout(() => this.finalizeChunk(), this.timeoutMs); + } + } + + protected finalizeChunk(): void { + if (!this.chunk) return; + const chunk = this.chunk; + this.chunk = null; + + if (this.chunkTimeout) { + clearTimeout(this.chunkTimeout); + this.chunkTimeout = null; + } + + // Discard empty chunks + if (!chunk.content.content && !chunk.content.embeds?.length) return; + + if (this.autoConsumeFn) { + this.autoConsumeFn(chunk.content); + return; + } + + this.finalizedChunks.push(chunk.content); + } + + consume(): StrictMessageContent[] { + return Array.from(this.finalizedChunks); + this.finalizedChunks = []; + } + + finalizeAndConsume(): StrictMessageContent[] { + this.finalizeChunk(); + return this.consume(); + } +}