mirror of
https://github.com/ZeppelinBot/Zeppelin.git
synced 2025-03-16 14:11:50 +00:00
Batch embed logs. Always use log batching.
This commit is contained in:
parent
0b7a5dbfbc
commit
001b6d00ea
4 changed files with 225 additions and 86 deletions
|
@ -266,7 +266,7 @@ export const LogsPlugin = zeppelinGuildPlugin<LogsPluginType>()({
|
||||||
state.archives = GuildArchives.getGuildInstance(guild.id);
|
state.archives = GuildArchives.getGuildInstance(guild.id);
|
||||||
state.cases = GuildCases.getGuildInstance(guild.id);
|
state.cases = GuildCases.getGuildInstance(guild.id);
|
||||||
|
|
||||||
state.batches = new Map();
|
state.buffers = new Map();
|
||||||
|
|
||||||
state.regexRunner = getRegExpRunner(`guild-${pluginData.guild.id}`);
|
state.regexRunner = getRegExpRunner(`guild-${pluginData.guild.id}`);
|
||||||
},
|
},
|
||||||
|
|
|
@ -6,7 +6,7 @@ import { GuildCases } from "../../data/GuildCases";
|
||||||
import { GuildLogs } from "../../data/GuildLogs";
|
import { GuildLogs } from "../../data/GuildLogs";
|
||||||
import { GuildSavedMessages } from "../../data/GuildSavedMessages";
|
import { GuildSavedMessages } from "../../data/GuildSavedMessages";
|
||||||
import { RegExpRunner } from "../../RegExpRunner";
|
import { RegExpRunner } from "../../RegExpRunner";
|
||||||
import { tMessageContent, tNullable } from "../../utils";
|
import { StrictMessageContent, tMessageContent, tNullable } from "../../utils";
|
||||||
import { TRegex } from "../../validatorUtils";
|
import { TRegex } from "../../validatorUtils";
|
||||||
import { LogType } from "../../data/LogType";
|
import { LogType } from "../../data/LogType";
|
||||||
import { GuildMember } from "discord.js";
|
import { GuildMember } from "discord.js";
|
||||||
|
@ -28,6 +28,7 @@ import {
|
||||||
TemplateSafeValueContainer,
|
TemplateSafeValueContainer,
|
||||||
TypedTemplateSafeValueContainer,
|
TypedTemplateSafeValueContainer,
|
||||||
} from "../../templateFormatter";
|
} from "../../templateFormatter";
|
||||||
|
import { MessageBuffer } from "../../utils/MessageBuffer";
|
||||||
|
|
||||||
export const tLogFormats = t.record(t.string, t.union([t.string, tMessageContent]));
|
export const tLogFormats = t.record(t.string, t.union([t.string, tMessageContent]));
|
||||||
export type TLogFormats = t.TypeOf<typeof tLogFormats>;
|
export type TLogFormats = t.TypeOf<typeof tLogFormats>;
|
||||||
|
@ -85,7 +86,7 @@ export interface LogsPluginType extends BasePluginType {
|
||||||
|
|
||||||
logListener;
|
logListener;
|
||||||
|
|
||||||
batches: Map<string, string[]>;
|
buffers: Map<string, MessageBuffer>;
|
||||||
|
|
||||||
onMessageDeleteFn;
|
onMessageDeleteFn;
|
||||||
onMessageDeleteBulkFn;
|
onMessageDeleteBulkFn;
|
||||||
|
|
|
@ -1,12 +1,11 @@
|
||||||
import { MessageMentionTypes, Snowflake, TextChannel } from "discord.js";
|
import { MessageEmbedOptions, MessageMentionTypes, Snowflake, TextChannel } from "discord.js";
|
||||||
import { GuildPluginData } from "knub";
|
import { GuildPluginData } from "knub";
|
||||||
import { SavedMessage } from "../../../data/entities/SavedMessage";
|
|
||||||
import { allowTimeout } from "../../../RegExpRunner";
|
import { allowTimeout } from "../../../RegExpRunner";
|
||||||
import { createChunkedMessage, get, noop } from "../../../utils";
|
import { ILogTypeData, LogsPluginType, TLogChannel, TLogChannelMap } from "../types";
|
||||||
import { ILogTypeData, LogsPluginType, LogTypeData, TLogChannelMap } from "../types";
|
|
||||||
import { getLogMessage } from "./getLogMessage";
|
import { getLogMessage } from "./getLogMessage";
|
||||||
import { TemplateSafeValueContainer, TypedTemplateSafeValueContainer } from "../../../templateFormatter";
|
import { TypedTemplateSafeValueContainer } from "../../../templateFormatter";
|
||||||
import { LogType } from "../../../data/LogType";
|
import { LogType } from "../../../data/LogType";
|
||||||
|
import { MessageBuffer } from "../../../utils/MessageBuffer";
|
||||||
|
|
||||||
const excludedUserProps = ["user", "member", "mod"];
|
const excludedUserProps = ["user", "member", "mod"];
|
||||||
const excludedRoleProps = ["message.member.roles", "member.roles"];
|
const excludedRoleProps = ["message.member.roles", "member.roles"];
|
||||||
|
@ -24,6 +23,53 @@ interface ExclusionData {
|
||||||
messageTextContent?: string | null;
|
messageTextContent?: string | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const DEFAULT_BATCH_TIME = 1000;
|
||||||
|
const MIN_BATCH_TIME = 250;
|
||||||
|
const MAX_BATCH_TIME = 5000;
|
||||||
|
|
||||||
|
async function shouldExclude(
|
||||||
|
pluginData: GuildPluginData<LogsPluginType>,
|
||||||
|
opts: TLogChannel,
|
||||||
|
exclusionData: ExclusionData,
|
||||||
|
): Promise<boolean> {
|
||||||
|
if (opts.excluded_users && exclusionData.userId && opts.excluded_users.includes(exclusionData.userId)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.exclude_bots && exclusionData.bot) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.excluded_roles && exclusionData.roles) {
|
||||||
|
for (const role of exclusionData.roles) {
|
||||||
|
if (opts.excluded_roles.includes(role)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.excluded_channels && exclusionData.channel && opts.excluded_channels.includes(exclusionData.channel)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.excluded_categories && exclusionData.category && opts.excluded_categories.includes(exclusionData.category)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.excluded_message_regexes && exclusionData.messageTextContent) {
|
||||||
|
for (const regex of opts.excluded_message_regexes) {
|
||||||
|
const matches = await pluginData.state.regexRunner
|
||||||
|
.exec(regex, exclusionData.messageTextContent)
|
||||||
|
.catch(allowTimeout);
|
||||||
|
if (matches) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
export async function log<TLogType extends keyof ILogTypeData>(
|
export async function log<TLogType extends keyof ILogTypeData>(
|
||||||
pluginData: GuildPluginData<LogsPluginType>,
|
pluginData: GuildPluginData<LogsPluginType>,
|
||||||
type: TLogType,
|
type: TLogType,
|
||||||
|
@ -36,86 +82,47 @@ export async function log<TLogType extends keyof ILogTypeData>(
|
||||||
logChannelLoop: for (const [channelId, opts] of Object.entries(logChannels)) {
|
logChannelLoop: for (const [channelId, opts] of Object.entries(logChannels)) {
|
||||||
const channel = pluginData.guild.channels.cache.get(channelId as Snowflake);
|
const channel = pluginData.guild.channels.cache.get(channelId as Snowflake);
|
||||||
if (!channel || !(channel instanceof TextChannel)) continue;
|
if (!channel || !(channel instanceof TextChannel)) continue;
|
||||||
|
if (opts.include?.length && !opts.include.includes(typeStr)) continue;
|
||||||
|
if (opts.exclude && opts.exclude.includes(typeStr)) continue;
|
||||||
|
if (await shouldExclude(pluginData, opts, exclusionData)) continue;
|
||||||
|
|
||||||
if ((opts.include && opts.include.includes(typeStr)) || (opts.exclude && !opts.exclude.includes(typeStr))) {
|
const message = await getLogMessage(pluginData, type, data, {
|
||||||
// If this log entry is about an excluded user, skip it
|
format: opts.format,
|
||||||
// TODO: Quick and dirty solution, look into changing at some point
|
include_embed_timestamp: opts.include_embed_timestamp,
|
||||||
if (opts.excluded_users && exclusionData.userId && opts.excluded_users.includes(exclusionData.userId)) {
|
timestamp_format: opts.timestamp_format,
|
||||||
continue;
|
});
|
||||||
}
|
if (!message) return;
|
||||||
|
|
||||||
// If we're excluding bots and the logged user is a bot, skip it
|
// Initialize message buffer for this channel
|
||||||
if (opts.exclude_bots && exclusionData.bot) {
|
if (!pluginData.state.buffers.has(channelId)) {
|
||||||
continue;
|
const batchTime = Math.min(Math.max(opts.batch_time ?? DEFAULT_BATCH_TIME, MIN_BATCH_TIME), MAX_BATCH_TIME);
|
||||||
}
|
pluginData.state.buffers.set(
|
||||||
|
channelId,
|
||||||
if (opts.excluded_roles && exclusionData.roles) {
|
new MessageBuffer({
|
||||||
for (const role of exclusionData.roles) {
|
timeout: batchTime,
|
||||||
if (opts.excluded_roles.includes(role)) {
|
consume: part => {
|
||||||
continue logChannelLoop;
|
const parse: MessageMentionTypes[] = pluginData.config.get().allow_user_mentions ? ["users"] : [];
|
||||||
}
|
channel
|
||||||
}
|
.send({
|
||||||
}
|
...part,
|
||||||
|
allowedMentions: { parse },
|
||||||
if (opts.excluded_channels && exclusionData.channel && opts.excluded_channels.includes(exclusionData.channel)) {
|
})
|
||||||
continue;
|
.catch(err => {
|
||||||
}
|
// tslint:disable-next-line:no-console
|
||||||
|
console.warn(
|
||||||
if (
|
`Error while sending ${typeStr} log to ${pluginData.guild.id}/${channelId}: ${err.message}`,
|
||||||
opts.excluded_categories &&
|
);
|
||||||
exclusionData.category &&
|
});
|
||||||
opts.excluded_categories.includes(exclusionData.category)
|
},
|
||||||
) {
|
}),
|
||||||
continue;
|
);
|
||||||
}
|
|
||||||
|
|
||||||
if (opts.excluded_message_regexes && exclusionData.messageTextContent) {
|
|
||||||
for (const regex of opts.excluded_message_regexes) {
|
|
||||||
const matches = await pluginData.state.regexRunner
|
|
||||||
.exec(regex, exclusionData.messageTextContent)
|
|
||||||
.catch(allowTimeout);
|
|
||||||
if (matches) {
|
|
||||||
continue logChannelLoop;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const message = await getLogMessage(pluginData, type, data, {
|
|
||||||
format: opts.format,
|
|
||||||
include_embed_timestamp: opts.include_embed_timestamp,
|
|
||||||
timestamp_format: opts.timestamp_format,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (message) {
|
|
||||||
// For non-string log messages (i.e. embeds) batching or chunking is not possible, so send them immediately
|
|
||||||
if (typeof message !== "string") {
|
|
||||||
await channel.send(message).catch(noop);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Default to batched unless explicitly disabled
|
|
||||||
const batched = opts.batched ?? true;
|
|
||||||
const batchTime = opts.batch_time ?? 1000;
|
|
||||||
const cfg = pluginData.config.get();
|
|
||||||
const parse: MessageMentionTypes[] = cfg.allow_user_mentions ? ["users"] : [];
|
|
||||||
|
|
||||||
if (batched) {
|
|
||||||
// If we're batching log messages, gather all log messages within the set batch_time into a single message
|
|
||||||
if (!pluginData.state.batches.has(channel.id)) {
|
|
||||||
pluginData.state.batches.set(channel.id, []);
|
|
||||||
setTimeout(async () => {
|
|
||||||
const batchedMessage = pluginData.state.batches.get(channel.id)!.join("\n");
|
|
||||||
pluginData.state.batches.delete(channel.id);
|
|
||||||
createChunkedMessage(channel, batchedMessage, { parse }).catch(noop);
|
|
||||||
}, batchTime);
|
|
||||||
}
|
|
||||||
|
|
||||||
pluginData.state.batches.get(channel.id)!.push(message);
|
|
||||||
} else {
|
|
||||||
// If we're not batching log messages, just send them immediately
|
|
||||||
await createChunkedMessage(channel, message, { parse }).catch(noop);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Add log message to buffer
|
||||||
|
const buffer = pluginData.state.buffers.get(channelId)!;
|
||||||
|
buffer.push({
|
||||||
|
content: typeof message === "string" ? message : message.content || "",
|
||||||
|
embeds: typeof message === "string" ? [] : ((message.embeds || []) as MessageEmbedOptions[]),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
131
backend/src/utils/MessageBuffer.ts
Normal file
131
backend/src/utils/MessageBuffer.ts
Normal file
|
@ -0,0 +1,131 @@
|
||||||
|
import { StrictMessageContent } from "../utils";
|
||||||
|
import Timeout = NodeJS.Timeout;
|
||||||
|
|
||||||
|
type ConsumeFn = (part: StrictMessageContent) => void;
|
||||||
|
|
||||||
|
type ContentType = "mixed" | "plain" | "embeds";
|
||||||
|
|
||||||
|
export type MessageBufferContent = Pick<StrictMessageContent, "content" | "embeds">;
|
||||||
|
|
||||||
|
type Chunk = {
|
||||||
|
type: ContentType;
|
||||||
|
content: MessageBufferContent;
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface MessageBufferOpts {
|
||||||
|
consume?: ConsumeFn;
|
||||||
|
timeout?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
const MAX_CHARS_PER_MESSAGE = 2000;
|
||||||
|
const MAX_EMBEDS_PER_MESSAGE = 10;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Allows buffering and automatic partitioning of message contents. Useful for e.g. high volume log channels, message chunking, etc.
|
||||||
|
*/
|
||||||
|
export class MessageBuffer {
|
||||||
|
protected autoConsumeFn: ConsumeFn | null = null;
|
||||||
|
|
||||||
|
protected timeoutMs: number | null = null;
|
||||||
|
|
||||||
|
protected chunk: Chunk | null = null;
|
||||||
|
|
||||||
|
protected chunkTimeout: Timeout | null = null;
|
||||||
|
|
||||||
|
protected finalizedChunks: MessageBufferContent[] = [];
|
||||||
|
|
||||||
|
constructor(opts: MessageBufferOpts = {}) {
|
||||||
|
if (opts.consume) {
|
||||||
|
this.autoConsumeFn = opts.consume;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.timeout) {
|
||||||
|
this.timeoutMs = opts.timeout;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
push(content: MessageBufferContent): void {
|
||||||
|
let contentType: ContentType;
|
||||||
|
if (content.content && !content.embeds?.length) {
|
||||||
|
contentType = "plain";
|
||||||
|
} else if (content.embeds?.length && !content.content) {
|
||||||
|
contentType = "embeds";
|
||||||
|
} else {
|
||||||
|
contentType = "mixed";
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plain text can't be merged with mixed or embeds
|
||||||
|
if (contentType === "plain" && this.chunk && this.chunk.type !== "plain") {
|
||||||
|
this.startNewChunk(contentType);
|
||||||
|
}
|
||||||
|
// Mixed can't be merged at all
|
||||||
|
if (contentType === "mixed" && this.chunk) {
|
||||||
|
this.startNewChunk(contentType);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!this.chunk) this.startNewChunk(contentType);
|
||||||
|
const chunk = this.chunk!;
|
||||||
|
|
||||||
|
if (content.content) {
|
||||||
|
if (chunk.content.content && chunk.content.content.length + content.content.length > MAX_CHARS_PER_MESSAGE) {
|
||||||
|
this.startNewChunk(contentType);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (chunk.content.content == null) chunk.content.content = "";
|
||||||
|
chunk.content.content += content.content;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (content.embeds) {
|
||||||
|
if (chunk.content.embeds && chunk.content.embeds.length + content.embeds.length > MAX_EMBEDS_PER_MESSAGE) {
|
||||||
|
this.startNewChunk(contentType);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (chunk.content.embeds == null) chunk.content.embeds = [];
|
||||||
|
chunk.content.embeds.push(...content.embeds);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected startNewChunk(type: ContentType): void {
|
||||||
|
if (this.chunk) {
|
||||||
|
this.finalizeChunk();
|
||||||
|
}
|
||||||
|
this.chunk = {
|
||||||
|
type,
|
||||||
|
content: {},
|
||||||
|
};
|
||||||
|
if (this.timeoutMs) {
|
||||||
|
this.chunkTimeout = setTimeout(() => this.finalizeChunk(), this.timeoutMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
protected finalizeChunk(): void {
|
||||||
|
if (!this.chunk) return;
|
||||||
|
const chunk = this.chunk;
|
||||||
|
this.chunk = null;
|
||||||
|
|
||||||
|
if (this.chunkTimeout) {
|
||||||
|
clearTimeout(this.chunkTimeout);
|
||||||
|
this.chunkTimeout = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discard empty chunks
|
||||||
|
if (!chunk.content.content && !chunk.content.embeds?.length) return;
|
||||||
|
|
||||||
|
if (this.autoConsumeFn) {
|
||||||
|
this.autoConsumeFn(chunk.content);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.finalizedChunks.push(chunk.content);
|
||||||
|
}
|
||||||
|
|
||||||
|
consume(): StrictMessageContent[] {
|
||||||
|
return Array.from(this.finalizedChunks);
|
||||||
|
this.finalizedChunks = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
finalizeAndConsume(): StrictMessageContent[] {
|
||||||
|
this.finalizeChunk();
|
||||||
|
return this.consume();
|
||||||
|
}
|
||||||
|
}
|
Loading…
Add table
Reference in a new issue