diff --git a/backend/src/api/index.ts b/backend/src/api/index.ts index 575b8b51..97049892 100644 --- a/backend/src/api/index.ts +++ b/backend/src/api/index.ts @@ -1,5 +1,6 @@ import { connect } from "../data/db"; import path from "path"; +import { setIsAPI } from "../globals"; require("dotenv").config({ path: path.resolve(process.cwd(), "api.env") }); @@ -10,6 +11,8 @@ function errorHandler(err) { process.on("unhandledRejection", errorHandler); +setIsAPI(true); + // Connect to the database before loading the rest of the code (that depend on the database connection) console.log("Connecting to database..."); // tslint:disable-line connect().then(() => { diff --git a/backend/src/data/Configs.ts b/backend/src/data/Configs.ts index 236713a4..054ce255 100644 --- a/backend/src/data/Configs.ts +++ b/backend/src/data/Configs.ts @@ -1,15 +1,22 @@ import { Config } from "./entities/Config"; -import { - getConnection, - getRepository, - Repository, - Transaction, - TransactionManager, - TransactionRepository, -} from "typeorm"; -import { BaseGuildRepository } from "./BaseGuildRepository"; +import { getRepository, Repository } from "typeorm"; import { connection } from "./db"; import { BaseRepository } from "./BaseRepository"; +import { isAPI } from "../globals"; +import { HOURS, SECONDS } from "../utils"; +import { cleanupConfigs } from "./cleanup/configs"; + +if (isAPI()) { + const CLEANUP_INTERVAL = 1 * HOURS; + + async function cleanup() { + await cleanupConfigs(); + setTimeout(cleanup, CLEANUP_INTERVAL); + } + + // Start first cleanup 30 seconds after startup + setTimeout(cleanup, 30 * SECONDS); +} export class Configs extends BaseRepository { private configs: Repository; diff --git a/backend/src/data/GuildNicknameHistory.ts b/backend/src/data/GuildNicknameHistory.ts index 58ed7273..91d9d533 100644 --- a/backend/src/data/GuildNicknameHistory.ts +++ b/backend/src/data/GuildNicknameHistory.ts @@ -1,7 +1,22 @@ import { BaseGuildRepository } from "./BaseGuildRepository"; -import { getRepository, Repository } from "typeorm"; +import { getRepository, In, Repository } from "typeorm"; import { NicknameHistoryEntry } from "./entities/NicknameHistoryEntry"; -import { sorter } from "../utils"; +import { MINUTES, SECONDS, sorter } from "../utils"; +import { MAX_USERNAME_ENTRIES_PER_USER } from "./UsernameHistory"; +import { isAPI } from "../globals"; +import { cleanupNicknames } from "./cleanup/nicknames"; + +if (!isAPI()) { + const CLEANUP_INTERVAL = 5 * MINUTES; + + async function cleanup() { + await cleanupNicknames(); + setTimeout(cleanup, CLEANUP_INTERVAL); + } + + // Start first cleanup 30 seconds after startup + setTimeout(cleanup, 30 * SECONDS); +} export const MAX_NICKNAME_ENTRIES_PER_USER = 10; @@ -44,25 +59,20 @@ export class GuildNicknameHistory extends BaseGuildRepository { nickname, }); - // Cleanup (leave only the last MAX_NICKNAME_ENTRIES_PER_USER entries) - const lastEntries = await this.getByUserId(userId); - if (lastEntries.length > MAX_NICKNAME_ENTRIES_PER_USER) { - const earliestEntry = lastEntries - .sort(sorter("timestamp", "DESC")) - .slice(0, 10) - .reduce((earliest, entry) => { - if (earliest == null) return entry; - if (entry.id < earliest.id) return entry; - return earliest; - }, null); + // Cleanup (leave only the last MAX_USERNAME_ENTRIES_PER_USER entries) + const toDelete = await this.nicknameHistory + .createQueryBuilder() + .where("guild_id = :guildId", { guildId: this.guildId }) + .andWhere("user_id = :userId", { userId }) + .orderBy("id", "DESC") + .skip(MAX_USERNAME_ENTRIES_PER_USER) + .take(99_999) + .getMany(); - this.nicknameHistory - .createQueryBuilder() - .where("guild_id = :guildId", { guildId: this.guildId }) - .andWhere("user_id = :userId", { userId }) - .andWhere("id < :id", { id: earliestEntry.id }) - .delete() - .execute(); + if (toDelete.length > 0) { + await this.nicknameHistory.delete({ + id: In(toDelete.map(v => v.id)), + }); } } } diff --git a/backend/src/data/GuildSavedMessages.ts b/backend/src/data/GuildSavedMessages.ts index 8507a0dd..76047dcd 100644 --- a/backend/src/data/GuildSavedMessages.ts +++ b/backend/src/data/GuildSavedMessages.ts @@ -1,41 +1,26 @@ -import { Brackets, getRepository, Repository } from "typeorm"; +import { getRepository, In, Repository } from "typeorm"; import { BaseGuildRepository } from "./BaseGuildRepository"; import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage"; import { QueuedEventEmitter } from "../QueuedEventEmitter"; import { GuildChannel, Message } from "eris"; import moment from "moment-timezone"; +import { DAYS, DBDateFormat, MINUTES, SECONDS } from "../utils"; +import { isAPI } from "../globals"; +import { connection } from "./db"; +import { cleanupMessages } from "./cleanup/messages"; -const CLEANUP_INTERVAL = 5 * 60 * 1000; // 5 min +if (!isAPI()) { + const CLEANUP_INTERVAL = 5 * MINUTES; -const RETENTION_PERIOD = 5 * 24 * 60 * 60 * 1000; // 5 days + async function cleanup() { + await cleanupMessages(); + setTimeout(cleanup, CLEANUP_INTERVAL); + } -async function cleanup() { - const repository = getRepository(SavedMessage); - await repository - .createQueryBuilder("messages") - .where( - // Clear deleted messages - new Brackets(qb => { - qb.where("deleted_at IS NOT NULL"); - qb.andWhere(`deleted_at <= (NOW() - INTERVAL ${CLEANUP_INTERVAL}000 MICROSECOND)`); - }), - ) - .orWhere( - // Clear old messages - new Brackets(qb => { - qb.where("is_permanent = 0"); - qb.andWhere(`posted_at <= (NOW() - INTERVAL ${RETENTION_PERIOD}000 MICROSECOND)`); - }), - ) - .delete() - .execute(); - - setTimeout(cleanup, CLEANUP_INTERVAL); + // Start first cleanup 30 seconds after startup + setTimeout(cleanup, 30 * SECONDS); } -// Start first cleanup 30 seconds after startup -setTimeout(cleanup, 30 * 1000); - export class GuildSavedMessages extends BaseGuildRepository { private messages: Repository; protected toBePermanent: Set; @@ -115,8 +100,8 @@ export class GuildSavedMessages extends BaseGuildRepository { let query = this.messages .createQueryBuilder() .where("guild_id = :guild_id", { guild_id: this.guildId }) - .andWhere("user_id = :user_id", { user_id: userId }) .andWhere("channel_id = :channel_id", { channel_id: channelId }) + .andWhere("user_id = :user_id", { user_id: userId }) .andWhere("id > :afterId", { afterId }) .andWhere("deleted_at IS NULL"); @@ -159,7 +144,7 @@ export class GuildSavedMessages extends BaseGuildRepository { if (existingSavedMsg) return; const savedMessageData = this.msgToSavedMessageData(msg); - const postedAt = moment.utc(msg.timestamp, "x").format("YYYY-MM-DD HH:mm:ss.SSS"); + const postedAt = moment.utc(msg.timestamp, "x").format("YYYY-MM-DD HH:mm:ss"); const data = { id: msg.id, @@ -198,7 +183,7 @@ export class GuildSavedMessages extends BaseGuildRepository { * If any messages were marked as deleted, also emits the deleteBulk event. */ async markBulkAsDeleted(ids) { - const deletedAt = moment().format("YYYY-MM-DD HH:mm:ss.SSS"); + const deletedAt = moment().format("YYYY-MM-DD HH:mm:ss"); await this.messages .createQueryBuilder() diff --git a/backend/src/data/Supporters.ts b/backend/src/data/Supporters.ts new file mode 100644 index 00000000..26c294c7 --- /dev/null +++ b/backend/src/data/Supporters.ts @@ -0,0 +1,16 @@ +import { BaseRepository } from "./BaseRepository"; +import { getRepository, Repository } from "typeorm"; +import { Supporter } from "./entities/Supporter"; + +export class Supporters extends BaseRepository { + private supporters: Repository; + + constructor() { + super(); + this.supporters = getRepository(Supporter); + } + + getAll() { + return this.supporters.find(); + } +} diff --git a/backend/src/data/UsernameHistory.ts b/backend/src/data/UsernameHistory.ts index e97d58cb..0d334adb 100644 --- a/backend/src/data/UsernameHistory.ts +++ b/backend/src/data/UsernameHistory.ts @@ -1,9 +1,24 @@ -import { getRepository, Repository } from "typeorm"; +import { getRepository, In, Repository } from "typeorm"; import { UsernameHistoryEntry } from "./entities/UsernameHistoryEntry"; -import { sorter } from "../utils"; +import { MINUTES, SECONDS, sorter } from "../utils"; import { BaseRepository } from "./BaseRepository"; +import { connection } from "./db"; +import { isAPI } from "../globals"; +import { cleanupUsernames } from "./cleanup/usernames"; -export const MAX_USERNAME_ENTRIES_PER_USER = 10; +if (!isAPI()) { + const CLEANUP_INTERVAL = 5 * MINUTES; + + async function cleanup() { + await cleanupUsernames(); + setTimeout(cleanup, CLEANUP_INTERVAL); + } + + // Start first cleanup 30 seconds after startup + setTimeout(cleanup, 30 * SECONDS); +} + +export const MAX_USERNAME_ENTRIES_PER_USER = 5; export class UsernameHistory extends BaseRepository { private usernameHistory: Repository; @@ -43,23 +58,18 @@ export class UsernameHistory extends BaseRepository { }); // Cleanup (leave only the last MAX_USERNAME_ENTRIES_PER_USER entries) - const lastEntries = await this.getByUserId(userId); - if (lastEntries.length > MAX_USERNAME_ENTRIES_PER_USER) { - const earliestEntry = lastEntries - .sort(sorter("timestamp", "DESC")) - .slice(0, 10) - .reduce((earliest, entry) => { - if (earliest == null) return entry; - if (entry.id < earliest.id) return entry; - return earliest; - }, null); + const toDelete = await this.usernameHistory + .createQueryBuilder() + .where("user_id = :userId", { userId }) + .orderBy("id", "DESC") + .skip(MAX_USERNAME_ENTRIES_PER_USER) + .take(99_999) + .getMany(); - this.usernameHistory - .createQueryBuilder() - .andWhere("user_id = :userId", { userId }) - .andWhere("id < :id", { id: earliestEntry.id }) - .delete() - .execute(); + if (toDelete.length > 0) { + await this.usernameHistory.delete({ + id: In(toDelete.map(v => v.id)), + }); } } } diff --git a/backend/src/data/cleanup/configs.ts b/backend/src/data/cleanup/configs.ts new file mode 100644 index 00000000..4775b74f --- /dev/null +++ b/backend/src/data/cleanup/configs.ts @@ -0,0 +1,96 @@ +import { connection } from "../db"; +import { getRepository, In } from "typeorm"; +import { Config } from "../entities/Config"; +import moment from "moment-timezone"; +import { DBDateFormat } from "../../utils"; + +const CLEAN_PER_LOOP = 50; + +export async function cleanupConfigs() { + const configRepository = getRepository(Config); + + let cleaned = 0; + let rows; + + // >1 month old: 1 config retained per month + const oneMonthCutoff = moment() + .subtract(30, "days") + .format(DBDateFormat); + do { + rows = await connection.query( + ` + WITH _configs + AS ( + SELECT + id, + \`key\`, + YEAR(edited_at) AS \`year\`, + MONTH(edited_at) AS \`month\`, + ROW_NUMBER() OVER ( + PARTITION BY \`key\`, \`year\`, \`month\` + ORDER BY edited_at + ) AS row_num + FROM + configs + WHERE + is_active = 0 + AND edited_at < ? + ) + SELECT * + FROM _configs + WHERE row_num > 1 + `, + [oneMonthCutoff], + ); + + if (rows.length > 0) { + await configRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + // >2 weeks old: 1 config retained per day + const twoWeekCutoff = moment() + .subtract(2, "weeks") + .format(DBDateFormat); + do { + rows = await connection.query( + ` + WITH _configs + AS ( + SELECT + id, + \`key\`, + DATE(edited_at) AS \`date\`, + ROW_NUMBER() OVER ( + PARTITION BY \`key\`, \`date\` + ORDER BY edited_at + ) AS row_num + FROM + configs + WHERE + is_active = 0 + AND edited_at < ? + AND edited_at >= ? + ) + SELECT * + FROM _configs + WHERE row_num > 1 + `, + [twoWeekCutoff, oneMonthCutoff], + ); + + if (rows.length > 0) { + await configRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + return cleaned; +} diff --git a/backend/src/data/cleanup/messages.ts b/backend/src/data/cleanup/messages.ts new file mode 100644 index 00000000..ddb42232 --- /dev/null +++ b/backend/src/data/cleanup/messages.ts @@ -0,0 +1,68 @@ +import { DAYS, DBDateFormat, MINUTES } from "../../utils"; +import { getRepository, In } from "typeorm"; +import { SavedMessage } from "../entities/SavedMessage"; +import moment from "moment-timezone"; +import { connection } from "../db"; + +/** + * How long message edits, deletions, etc. will include the original message content. + * This is very heavy storage-wise, so keeping it as low as possible is ideal. + */ +const RETENTION_PERIOD = 1 * DAYS; +const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES; +const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES; +const CLEAN_PER_LOOP = 500; + +export async function cleanupMessages(): Promise { + let cleaned = 0; + + const messagesRepository = getRepository(SavedMessage); + + const deletedAtThreshold = moment() + .subtract(DELETED_MESSAGE_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + const postedAtThreshold = moment() + .subtract(RETENTION_PERIOD, "ms") + .format(DBDateFormat); + const botPostedAtThreshold = moment() + .subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + + // SELECT + DELETE messages in batches + // This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below + // when a message was being inserted at the same time + let rows; + do { + rows = await connection.query( + ` + SELECT id + FROM messages + WHERE ( + deleted_at IS NOT NULL + AND deleted_at <= ? + ) + OR ( + posted_at <= ? + AND is_permanent = 0 + ) + OR ( + is_bot = 1 + AND posted_at <= ? + AND is_permanent = 0 + ) + LIMIT ${CLEAN_PER_LOOP} + `, + [deletedAtThreshold, postedAtThreshold, botPostedAtThreshold], + ); + + if (rows.length > 0) { + await messagesRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + return cleaned; +} diff --git a/backend/src/data/cleanup/nicknames.ts b/backend/src/data/cleanup/nicknames.ts new file mode 100644 index 00000000..3f41084d --- /dev/null +++ b/backend/src/data/cleanup/nicknames.ts @@ -0,0 +1,41 @@ +import { getRepository, In } from "typeorm"; +import moment from "moment-timezone"; +import { NicknameHistoryEntry } from "../entities/NicknameHistoryEntry"; +import { DAYS, DBDateFormat } from "../../utils"; +import { connection } from "../db"; + +export const NICKNAME_RETENTION_PERIOD = 30 * DAYS; +const CLEAN_PER_LOOP = 500; + +export async function cleanupNicknames(): Promise { + let cleaned = 0; + + const nicknameHistoryRepository = getRepository(NicknameHistoryEntry); + const dateThreshold = moment() + .subtract(NICKNAME_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + + // Clean old nicknames (NICKNAME_RETENTION_PERIOD) + let rows; + do { + rows = await connection.query( + ` + SELECT id + FROM nickname_history + WHERE timestamp < ? + LIMIT ${CLEAN_PER_LOOP} + `, + [dateThreshold], + ); + + if (rows.length > 0) { + await nicknameHistoryRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + return cleaned; +} diff --git a/backend/src/data/cleanup/usernames.ts b/backend/src/data/cleanup/usernames.ts new file mode 100644 index 00000000..71afcfbc --- /dev/null +++ b/backend/src/data/cleanup/usernames.ts @@ -0,0 +1,41 @@ +import { getRepository, In } from "typeorm"; +import moment from "moment-timezone"; +import { UsernameHistoryEntry } from "../entities/UsernameHistoryEntry"; +import { DAYS, DBDateFormat } from "../../utils"; +import { connection } from "../db"; + +export const USERNAME_RETENTION_PERIOD = 30 * DAYS; +const CLEAN_PER_LOOP = 500; + +export async function cleanupUsernames(): Promise { + let cleaned = 0; + + const usernameHistoryRepository = getRepository(UsernameHistoryEntry); + const dateThreshold = moment() + .subtract(USERNAME_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + + // Clean old usernames (USERNAME_RETENTION_PERIOD) + let rows; + do { + rows = await connection.query( + ` + SELECT id + FROM username_history + WHERE timestamp < ? + LIMIT ${CLEAN_PER_LOOP} + `, + [dateThreshold], + ); + + if (rows.length > 0) { + await usernameHistoryRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + return cleaned; +} diff --git a/backend/src/data/entities/Supporter.ts b/backend/src/data/entities/Supporter.ts new file mode 100644 index 00000000..4189b087 --- /dev/null +++ b/backend/src/data/entities/Supporter.ts @@ -0,0 +1,14 @@ +import { Entity, Column, PrimaryColumn } from "typeorm"; + +@Entity("supporters") +export class Supporter { + @Column() + @PrimaryColumn() + user_id: string; + + @Column() + name: string; + + @Column() + amount: string | null; +} diff --git a/backend/src/globals.ts b/backend/src/globals.ts new file mode 100644 index 00000000..0c5abcd4 --- /dev/null +++ b/backend/src/globals.ts @@ -0,0 +1,9 @@ +let isAPIValue = false; + +export function isAPI() { + return isAPIValue; +} + +export function setIsAPI(value: boolean) { + isAPIValue = value; +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 1794ea69..38a7fdec 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -100,7 +100,7 @@ connect().then(async () => { client.on("debug", message => { if (message.includes(" 429 ")) { - logger.info(`[RATELIMITED] ${message}`); + logger.info(`[429] ${message}`); } }); diff --git a/backend/src/migrations/1590616691907-CreateSupportersTable.ts b/backend/src/migrations/1590616691907-CreateSupportersTable.ts new file mode 100644 index 00000000..16273fbf --- /dev/null +++ b/backend/src/migrations/1590616691907-CreateSupportersTable.ts @@ -0,0 +1,36 @@ +import { MigrationInterface, QueryRunner, Table } from "typeorm"; + +export class CreateSupportersTable1590616691907 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable( + new Table({ + name: "supporters", + columns: [ + { + name: "user_id", + type: "bigint", + unsigned: true, + isPrimary: true, + }, + { + name: "name", + type: "varchar", + length: "255", + }, + { + name: "amount", + type: "decimal", + precision: 6, + scale: 2, + isNullable: true, + default: null, + }, + ], + }), + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable("supporters"); + } +} diff --git a/backend/src/migrations/1591036185142-OptimizeMessageIndices.ts b/backend/src/migrations/1591036185142-OptimizeMessageIndices.ts new file mode 100644 index 00000000..46ecb42d --- /dev/null +++ b/backend/src/migrations/1591036185142-OptimizeMessageIndices.ts @@ -0,0 +1,45 @@ +import { MigrationInterface, QueryRunner, TableIndex } from "typeorm"; + +export class OptimizeMessageIndices1591036185142 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + // guild_id, channel_id, user_id indices -> composite(guild_id, channel_id, user_id) + await queryRunner.dropIndex("messages", "IDX_b193588441b085352a4c010942"); // guild_id + await queryRunner.dropIndex("messages", "IDX_86b9109b155eb70c0a2ca3b4b6"); // channel_id + await queryRunner.dropIndex("messages", "IDX_830a3c1d92614d1495418c4673"); // user_id + await queryRunner.createIndex( + "messages", + new TableIndex({ + columnNames: ["guild_id", "channel_id", "user_id"], + }), + ); + + // posted_at, is_permanent indices -> composite(posted_at, is_permanent) + await queryRunner.dropIndex("messages", "IDX_08e1f5a0fef0175ea402c6b2ac"); // posted_at + await queryRunner.dropIndex("messages", "IDX_f520029c07824f8d96c6cd98e8"); // is_permanent + await queryRunner.createIndex( + "messages", + new TableIndex({ + columnNames: ["posted_at", "is_permanent"], + }), + ); + + // is_bot -> no index (the database doesn't appear to use this index anyway) + await queryRunner.dropIndex("messages", "IDX_eec2c581ff6f13595902c31840"); + } + + public async down(queryRunner: QueryRunner): Promise { + // no index -> is_bot index + await queryRunner.createIndex("messages", new TableIndex({ columnNames: ["is_bot"] })); + + // composite(posted_at, is_permanent) -> posted_at, is_permanent indices + await queryRunner.dropIndex("messages", "IDX_afe125bfd65341cd90eee0b310"); // composite(posted_at, is_permanent) + await queryRunner.createIndex("messages", new TableIndex({ columnNames: ["posted_at"] })); + await queryRunner.createIndex("messages", new TableIndex({ columnNames: ["is_permanent"] })); + + // composite(guild_id, channel_id, user_id) -> guild_id, channel_id, user_id indices + await queryRunner.dropIndex("messages", "IDX_dedc3ea6396e1de8ac75533589"); // composite(guild_id, channel_id, user_id) + await queryRunner.createIndex("messages", new TableIndex({ columnNames: ["guild_id"] })); + await queryRunner.createIndex("messages", new TableIndex({ columnNames: ["channel_id"] })); + await queryRunner.createIndex("messages", new TableIndex({ columnNames: ["user_id"] })); + } +} diff --git a/backend/src/migrations/1591038041635-OptimizeMessageTimestamps.ts b/backend/src/migrations/1591038041635-OptimizeMessageTimestamps.ts new file mode 100644 index 00000000..80bdbc96 --- /dev/null +++ b/backend/src/migrations/1591038041635-OptimizeMessageTimestamps.ts @@ -0,0 +1,21 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class OptimizeMessageTimestamps1591038041635 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + // DATETIME(3) -> DATETIME(0) + await queryRunner.query(` + ALTER TABLE \`messages\` + CHANGE COLUMN \`posted_at\` \`posted_at\` DATETIME(0) NOT NULL AFTER \`data\`, + CHANGE COLUMN \`deleted_at\` \`deleted_at\` DATETIME(0) NULL DEFAULT NULL AFTER \`posted_at\` + `); + } + + public async down(queryRunner: QueryRunner): Promise { + // DATETIME(0) -> DATETIME(3) + await queryRunner.query(` + ALTER TABLE \`messages\` + CHANGE COLUMN \`posted_at\` \`posted_at\` DATETIME(3) NOT NULL AFTER \`data\`, + CHANGE COLUMN \`deleted_at\` \`deleted_at\` DATETIME(3) NULL DEFAULT NULL AFTER \`posted_at\` + `); + } +} diff --git a/backend/src/plugins/Automod/Automod.ts b/backend/src/plugins/Automod/Automod.ts index d68ad525..ca2a8678 100644 --- a/backend/src/plugins/Automod/Automod.ts +++ b/backend/src/plugins/Automod/Automod.ts @@ -125,12 +125,6 @@ const defaultMatchAttachmentTypeTrigger: Partial = blacklist_enabled: false, filetype_whitelist: [], whitelist_enabled: false, - match_messages: true, - match_embeds: true, - match_visible_names: false, - match_usernames: false, - match_nicknames: false, - match_custom_status: false, }; const defaultTextSpamTrigger: Partial> = { @@ -238,6 +232,7 @@ export class AutomodPlugin extends ZeppelinPluginClass this.onMessageCreate(msg); + this.onMessageCreateFn = msg => this.runAutomodOnMessage(msg, false); this.savedMessages.events.on("create", this.onMessageCreateFn); - this.savedMessages.events.on("update", this.onMessageCreateFn); + + this.onMessageUpdateFn = msg => this.runAutomodOnMessage(msg, true); + this.savedMessages.events.on("update", this.onMessageUpdateFn); + this.actionedMessageIds = []; } @@ -370,7 +372,7 @@ export class AutomodPlugin extends ZeppelinPluginClass { + protected async evaluateMatchInvitesTrigger(trigger: TMatchInvitesTrigger, str: string): Promise { const inviteCodes = getInviteCodesInString(str); if (inviteCodes.length === 0) return null; @@ -434,22 +436,22 @@ export class AutomodPlugin extends ZeppelinPluginClass { const since = moment.utc(msg.posted_at).valueOf() - convertDelayStringToMS(trigger.within); + const to = moment.utc(msg.posted_at).valueOf(); const identifier = trigger.per_channel ? `${msg.channel_id}-${msg.user_id}` : msg.user_id; - const recentActions = this.getMatchingRecentActions(recentActionType, identifier, since); + const recentActions = this.getMatchingRecentActions(recentActionType, identifier, since, to); const totalCount = recentActions.reduce((total, action) => { return total + action.count; }, 0); @@ -584,7 +596,8 @@ export class AutomodPlugin extends ZeppelinPluginClass { const since = moment.utc().valueOf() - convertDelayStringToMS(trigger.within); - const recentActions = this.getMatchingRecentActions(recentActionType, identifier, since) as OtherRecentAction[]; + const to = moment.utc().valueOf(); + const recentActions = this.getMatchingRecentActions(recentActionType, identifier, since, to) as OtherRecentAction[]; const totalCount = recentActions.reduce((total, action) => { return total + action.count; }, 0); @@ -729,10 +742,18 @@ export class AutomodPlugin extends ZeppelinPluginClass { - return this.evaluateMatchAttachmentTypeTrigger(trigger.match_attachment_type, msg); - }); - if (match) return { ...match, trigger: "match_attachment_type" } as TextTriggerMatchResult; + const match = this.evaluateMatchAttachmentTypeTrigger(trigger.match_attachment_type, msg); + // TODO: Add "attachment" type + if (match) { + const messageInfo: MessageInfo = { channelId: msg.channel_id, messageId: msg.id, userId: msg.user_id }; + return { + type: "message", + userId: msg.user_id, + messageInfo, + ...match, + trigger: "match_attachment_type", + }; + } } if (trigger.message_spam) { @@ -825,7 +846,7 @@ export class AutomodPlugin extends ZeppelinPluginClass { - return action.type === type && (!identifier || action.identifier === identifier) && action.timestamp >= since; + return ( + action.type === type && + (!identifier || action.identifier === identifier) && + action.timestamp >= since && + action.timestamp <= to + ); + }); + } + + protected clearRecentActionsForMessage(messageId: string) { + this.recentActions = this.recentActions.filter(info => { + return !((info as TextRecentAction).messageInfo?.messageId === messageId); }); } @@ -1495,7 +1527,10 @@ export class AutomodPlugin extends ZeppelinPluginClass { if (this.unloaded) return; - await this.logRecentActionsForMessage(msg); + if (isEdit) { + this.clearRecentActionsForMessage(msg.id); + } + this.logRecentActionsForMessage(msg); const member = this.guild.members.get(msg.user_id); const config = this.getMatchingConfig({ @@ -1524,6 +1561,8 @@ export class AutomodPlugin extends ZeppelinPluginClass { if (this.unloaded) return; @@ -1568,6 +1605,8 @@ export class AutomodPlugin extends ZeppelinPluginClass; @@ -302,6 +296,7 @@ export const Rule = t.type({ enabled: t.boolean, name: t.string, presets: tNullable(t.array(t.string)), + affects_bots: t.boolean, triggers: t.array( t.type({ match_words: tNullable(MatchWordsTrigger), diff --git a/backend/src/plugins/BotControl.ts b/backend/src/plugins/BotControl.ts index 44a82d87..6c246b6d 100644 --- a/backend/src/plugins/BotControl.ts +++ b/backend/src/plugins/BotControl.ts @@ -8,6 +8,7 @@ import { ZeppelinPluginClass } from "./ZeppelinPluginClass"; import { GuildArchives } from "../data/GuildArchives"; import { GlobalZeppelinPlugin } from "./GlobalZeppelinPlugin"; import * as t from "io-ts"; +import escapeStringRegexp from "escape-string-regexp"; let activeReload: [string, string] = null; @@ -121,22 +122,75 @@ export class BotControlPlugin extends GlobalZeppelinPlugin { } } - @d.command("guilds") + @d.command("guilds", "[search:string$]", { + aliases: ["servers"], + options: [ + { + name: "all", + shortcut: "a", + isSwitch: true, + }, + { + name: "initialized", + shortcut: "i", + isSwitch: true, + }, + { + name: "uninitialized", + shortcut: "u", + isSwitch: true, + }, + ], + }) @d.permission("can_use") - async serversCmd(msg: Message) { + async serversCmd( + msg: Message, + args: { search?: string; all?: boolean; initialized?: boolean; uninitialized?: boolean }, + ) { + const showList = Boolean(args.all || args.initialized || args.uninitialized || args.search); + const search = args.search && new RegExp([...args.search].map(s => escapeStringRegexp(s)).join(".*"), "i"); + const joinedGuilds = Array.from(this.bot.guilds.values()); const loadedGuilds = this.knub.getLoadedGuilds(); const loadedGuildsMap = loadedGuilds.reduce((map, guildData) => map.set(guildData.id, guildData), new Map()); - joinedGuilds.sort(sorter(g => g.name.toLowerCase())); - const longestId = joinedGuilds.reduce((longest, guild) => Math.max(longest, guild.id.length), 0); - const lines = joinedGuilds.map(g => { - const paddedId = g.id.padEnd(longestId, " "); - return `\`${paddedId}\` **${g.name}** (${loadedGuildsMap.has(g.id) ? "initialized" : "not initialized"}) (${ - g.memberCount - } members)`; - }); - createChunkedMessage(msg.channel, lines.join("\n")); + if (showList) { + let filteredGuilds = Array.from(joinedGuilds); + + if (args.initialized) { + filteredGuilds = filteredGuilds.filter(g => loadedGuildsMap.has(g.id)); + } + + if (args.uninitialized) { + filteredGuilds = filteredGuilds.filter(g => !loadedGuildsMap.has(g.id)); + } + + if (args.search) { + filteredGuilds = filteredGuilds.filter(g => search.test(`${g.id} ${g.name}`)); + } + + if (filteredGuilds.length) { + filteredGuilds.sort(sorter(g => g.name.toLowerCase())); + const longestId = filteredGuilds.reduce((longest, guild) => Math.max(longest, guild.id.length), 0); + const lines = filteredGuilds.map(g => { + const paddedId = g.id.padEnd(longestId, " "); + return `\`${paddedId}\` **${g.name}** (${loadedGuildsMap.has(g.id) ? "initialized" : "not initialized"}) (${ + g.memberCount + } members)`; + }); + createChunkedMessage(msg.channel, lines.join("\n")); + } else { + msg.channel.createMessage("No servers matched the filters"); + } + } else { + const total = joinedGuilds.length; + const initialized = joinedGuilds.filter(g => loadedGuildsMap.has(g.id)).length; + const unInitialized = total - initialized; + + msg.channel.createMessage( + `I am on **${total} total servers**, of which **${initialized} are initialized** and **${unInitialized} are not initialized**`, + ); + } } @d.command("leave_guild", "") diff --git a/backend/src/plugins/Cases.ts b/backend/src/plugins/Cases.ts index 2d9851a4..7fa96cbb 100644 --- a/backend/src/plugins/Cases.ts +++ b/backend/src/plugins/Cases.ts @@ -94,6 +94,14 @@ export class CasesPlugin extends ZeppelinPluginClass { ppName = `${pp.username}#${pp.discriminator}`; } + if (args.auditLogId) { + const existingAuditLogCase = await this.cases.findByAuditLogId(args.auditLogId); + if (existingAuditLogCase) { + delete args.auditLogId; + logger.warn(`Duplicate audit log ID for mod case: ${args.auditLogId}`); + } + } + const createdCase = await this.cases.create({ type: args.type, user_id: args.userId, @@ -275,7 +283,7 @@ export class CasesPlugin extends ZeppelinPluginClass { try { result = await caseLogChannel.createMessage(content, file); } catch (e) { - if (isDiscordRESTError(e) && e.code === 50013) { + if (isDiscordRESTError(e) && (e.code === 50013 || e.code === 50001)) { logger.warn( `Missing permissions to post mod cases in <#${caseLogChannel.id}> in guild ${this.guild.name} (${this.guild.id})`, ); diff --git a/backend/src/plugins/Logs.ts b/backend/src/plugins/Logs.ts index 1530c09d..a842a506 100644 --- a/backend/src/plugins/Logs.ts +++ b/backend/src/plugins/Logs.ts @@ -151,19 +151,14 @@ export class LogsPlugin extends ZeppelinPluginClass { // If this entry is from an excluded channel, skip it if (opts.excluded_channels) { - if (type === LogType.MESSAGE_DELETE || type === LogType.MESSAGE_DELETE_BARE) { - if (opts.excluded_channels.includes(data.message.channel.id)) { - continue logChannelLoop; - } - } - - if (type === LogType.MESSAGE_EDIT) { - if (opts.excluded_channels.includes(data.before.channel.id)) { - continue logChannelLoop; - } - } - - if (type === LogType.MESSAGE_SPAM_DETECTED || type === LogType.CENSOR || type === LogType.CLEAN) { + if ( + type === LogType.MESSAGE_DELETE || + type === LogType.MESSAGE_DELETE_BARE || + type === LogType.MESSAGE_EDIT || + type === LogType.MESSAGE_SPAM_DETECTED || + type === LogType.CENSOR || + type === LogType.CLEAN + ) { if (opts.excluded_channels.includes(data.channel.id)) { continue logChannelLoop; } diff --git a/backend/src/plugins/ModActions.ts b/backend/src/plugins/ModActions.ts index 2ed34337..a7b5d9c1 100644 --- a/backend/src/plugins/ModActions.ts +++ b/backend/src/plugins/ModActions.ts @@ -46,6 +46,7 @@ const ConfigSchema = t.type({ ban_message: tNullable(t.string), alert_on_rejoin: t.boolean, alert_channel: tNullable(t.string), + warn_notify_enabled: t.boolean, warn_notify_threshold: t.number, warn_notify_message: t.string, ban_delete_message_days: t.number, @@ -166,6 +167,7 @@ export class ModActionsPlugin extends ZeppelinPluginClass { ban_message: "You have been banned from the {guildName} server. Reason given: {reason}", alert_on_rejoin: false, alert_channel: null, + warn_notify_enabled: false, warn_notify_threshold: 5, warn_notify_message: "The user already has **{priorWarnings}** warnings!\n Please check their prior cases and assess whether or not to warn anyways.\n Proceed with the warning?", @@ -690,7 +692,7 @@ export class ModActionsPlugin extends ZeppelinPluginClass { const casesPlugin = this.getPlugin("cases"); const priorWarnAmount = await casesPlugin.getCaseTypeAmountForUserId(memberToWarn.id, CaseTypes.Warn); - if (priorWarnAmount >= config.warn_notify_threshold) { + if (config.warn_notify_enabled && priorWarnAmount >= config.warn_notify_threshold) { const tooManyWarningsMsg = await msg.channel.createMessage( config.warn_notify_message.replace("{priorWarnings}", `${priorWarnAmount}`), ); @@ -840,6 +842,9 @@ export class ModActionsPlugin extends ZeppelinPluginClass { this.sendErrorMessage(msg.channel, "Could not mute the user: unknown member"); } else { logger.error(`Failed to mute user ${user.id}: ${e.stack}`); + if (user.id == null) { + console.trace("[DEBUG] Null user.id for mute"); + } this.sendErrorMessage(msg.channel, "Could not mute the user"); } @@ -1042,7 +1047,7 @@ export class ModActionsPlugin extends ZeppelinPluginClass { if (!user) return this.sendErrorMessage(msg.channel, `User not found`); const memberToUnmute = await this.getMember(user.id); const mutesPlugin = this.getPlugin("mutes"); - const hasMuteRole = mutesPlugin.hasMutedRole(memberToUnmute); + const hasMuteRole = memberToUnmute && mutesPlugin.hasMutedRole(memberToUnmute); // Check if they're muted in the first place if (!(await this.mutes.isMuted(args.user)) && !hasMuteRole) { diff --git a/backend/src/plugins/Mutes.ts b/backend/src/plugins/Mutes.ts index a27fbf3a..659adc2a 100644 --- a/backend/src/plugins/Mutes.ts +++ b/backend/src/plugins/Mutes.ts @@ -150,8 +150,11 @@ export class MutesPlugin extends ZeppelinPluginClass { reason: string = null, muteOptions: MuteOptions = {}, ): Promise { + const lock = await this.locks.acquire(`mute-${userId}`); + const muteRole = this.getConfig().mute_role; if (!muteRole) { + lock.unlock(); this.throwRecoverablePluginError(ERRORS.NO_MUTE_ROLE_IN_CONFIG); } @@ -287,6 +290,8 @@ export class MutesPlugin extends ZeppelinPluginClass { }); } + lock.unlock(); + return { case: theCase, notifyResult, diff --git a/backend/src/plugins/NameHistory.ts b/backend/src/plugins/NameHistory.ts index d5de4f98..ebf979d2 100644 --- a/backend/src/plugins/NameHistory.ts +++ b/backend/src/plugins/NameHistory.ts @@ -1,10 +1,14 @@ import { decorators as d, IPluginOptions } from "knub"; import { GuildNicknameHistory, MAX_NICKNAME_ENTRIES_PER_USER } from "../data/GuildNicknameHistory"; import { Member, Message } from "eris"; -import { createChunkedMessage, disableCodeBlocks } from "../utils"; import { ZeppelinPluginClass } from "./ZeppelinPluginClass"; +import { createChunkedMessage, DAYS, disableCodeBlocks } from "../utils"; import { MAX_USERNAME_ENTRIES_PER_USER, UsernameHistory } from "../data/UsernameHistory"; import * as t from "io-ts"; +import { NICKNAME_RETENTION_PERIOD } from "../data/cleanup/nicknames"; +import moment from "moment-timezone"; +import { USERNAME_RETENTION_PERIOD } from "../data/cleanup/usernames"; +import { Queue } from "../Queue"; const ConfigSchema = t.type({ can_view: t.boolean, @@ -19,6 +23,8 @@ export class NameHistoryPlugin extends ZeppelinPluginClass { protected nicknameHistory: GuildNicknameHistory; protected usernameHistory: UsernameHistory; + protected updateQueue: Queue; + public static getStaticDefaultOptions(): IPluginOptions { return { config: { @@ -39,6 +45,7 @@ export class NameHistoryPlugin extends ZeppelinPluginClass { onLoad() { this.nicknameHistory = GuildNicknameHistory.getGuildInstance(this.guildId); this.usernameHistory = new UsernameHistory(); + this.updateQueue = new Queue(); } @d.command("names", "") @@ -59,23 +66,40 @@ export class NameHistoryPlugin extends ZeppelinPluginClass { const user = this.bot.users.get(args.userId); const currentUsername = user ? `${user.username}#${user.discriminator}` : args.userId; + const nicknameDays = Math.round(NICKNAME_RETENTION_PERIOD / DAYS); + const usernameDays = Math.round(USERNAME_RETENTION_PERIOD / DAYS); + let message = `Name history for **${currentUsername}**:`; if (nicknameRows.length) { - message += `\n\n__Last ${MAX_NICKNAME_ENTRIES_PER_USER} nicknames:__\n${nicknameRows.join("\n")}`; + message += `\n\n__Last ${MAX_NICKNAME_ENTRIES_PER_USER} nicknames within ${nicknameDays} days:__\n${nicknameRows.join( + "\n", + )}`; } if (usernameRows.length) { - message += `\n\n__Last ${MAX_USERNAME_ENTRIES_PER_USER} usernames:__\n${usernameRows.join("\n")}`; + message += `\n\n__Last ${MAX_USERNAME_ENTRIES_PER_USER} usernames within ${usernameDays} days:__\n${usernameRows.join( + "\n", + )}`; } createChunkedMessage(msg.channel, message); } - @d.event("guildMemberUpdate") - async onGuildMemberUpdate(_, member: Member) { + async updateNickname(member: Member) { + if (!member) return; const latestEntry = await this.nicknameHistory.getLastEntry(member.id); if (!latestEntry || latestEntry.nickname !== member.nick) { - // tslint:disable-line + if (!latestEntry && member.nick == null) return; // No need to save "no nickname" if there's no previous data await this.nicknameHistory.addEntry(member.id, member.nick); } } + + @d.event("messageCreate") + async onMessage(msg: Message) { + this.updateQueue.add(() => this.updateNickname(msg.member)); + } + + @d.event("voiceChannelJoin") + async onVoiceChannelJoin(member: Member) { + this.updateQueue.add(() => this.updateNickname(member)); + } } diff --git a/backend/src/plugins/ReactionRoles.ts b/backend/src/plugins/ReactionRoles.ts index e1c61749..99f79209 100644 --- a/backend/src/plugins/ReactionRoles.ts +++ b/backend/src/plugins/ReactionRoles.ts @@ -54,6 +54,7 @@ export class ReactionRolesPlugin extends ZeppelinPluginClass { protected savedMessages: GuildSavedMessages; protected reactionRemoveQueue: Queue; + protected roleChangeQueue: Queue; protected pendingRoleChanges: Map; protected pendingRefreshes: Set; @@ -82,6 +83,7 @@ export class ReactionRolesPlugin extends ZeppelinPluginClass { this.reactionRoles = GuildReactionRoles.getGuildInstance(this.guildId); this.savedMessages = GuildSavedMessages.getGuildInstance(this.guildId); this.reactionRemoveQueue = new Queue(); + this.roleChangeQueue = new Queue(); this.pendingRoleChanges = new Map(); this.pendingRefreshes = new Set(); @@ -202,16 +204,18 @@ export class ReactionRolesPlugin extends ZeppelinPluginClass { } try { - await member.edit({ - roles: Array.from(newRoleIds.values()), - }); + await member.edit( + { + roles: Array.from(newRoleIds.values()), + }, + "Reaction roles", + ); } catch (e) { logger.warn( `Failed to apply role changes to ${member.username}#${member.discriminator} (${member.id}): ${e.message}`, ); } } - lock.unlock(); }, }; @@ -223,7 +227,10 @@ export class ReactionRolesPlugin extends ZeppelinPluginClass { pendingRoleChangeObj.changes.push({ mode, roleId }); if (pendingRoleChangeObj.timeout) clearTimeout(pendingRoleChangeObj.timeout); - setTimeout(() => pendingRoleChangeObj.applyFn(), ROLE_CHANGE_BATCH_DEBOUNCE_TIME); + pendingRoleChangeObj.timeout = setTimeout( + () => this.roleChangeQueue.add(pendingRoleChangeObj.applyFn), + ROLE_CHANGE_BATCH_DEBOUNCE_TIME, + ); } /** diff --git a/backend/src/plugins/Roles.ts b/backend/src/plugins/Roles.ts index 81ab9312..e179b53e 100644 --- a/backend/src/plugins/Roles.ts +++ b/backend/src/plugins/Roles.ts @@ -1,6 +1,6 @@ import { trimPluginDescription, ZeppelinPluginClass } from "./ZeppelinPluginClass"; import * as t from "io-ts"; -import { resolveMember, stripObjectToScalars, successMessage } from "../utils"; +import { resolveMember, stripObjectToScalars, successMessage, verboseUserMention } from "../utils"; import { decorators as d, IPluginOptions, logger } from "knub"; import { GuildChannel, Member, Message } from "eris"; import { GuildLogs } from "../data/GuildLogs"; @@ -100,7 +100,7 @@ export class RolesPlugin extends ZeppelinPluginClass { mod: stripObjectToScalars(msg.author), }); - this.sendSuccessMessage(msg.channel, "Role added to user!"); + this.sendSuccessMessage(msg.channel, `Added role **${role.name}** to ${verboseUserMention(args.member.user)}!`); } @d.command("massaddrole", " ") @@ -129,19 +129,30 @@ export class RolesPlugin extends ZeppelinPluginClass { if (!roleId) { return this.sendErrorMessage(msg.channel, "Invalid role id"); } - const role = this.guild.roles.get(roleId); const config = this.getConfigForMsg(msg); if (!config.assignable_roles.includes(roleId)) { return this.sendErrorMessage(msg.channel, "You cannot assign that role"); } + const role = this.guild.roles.get(roleId); + if (!role) { + this.logs.log(LogType.BOT_ALERT, { + body: `Unknown role configured for 'roles' plugin: ${roleId}`, + }); + return this.sendErrorMessage(msg.channel, "You cannot assign that role"); + } + const membersWithoutTheRole = members.filter(m => !m.roles.includes(roleId)); let assigned = 0; const failed = []; const alreadyHadRole = members.length - membersWithoutTheRole.length; - msg.channel.createMessage(`Adding role to specified members...`); + msg.channel.createMessage( + `Adding role **${role.name}** to ${membersWithoutTheRole.length} ${ + membersWithoutTheRole.length === 1 ? "member" : "members" + }...`, + ); for (const member of membersWithoutTheRole) { try { @@ -159,7 +170,7 @@ export class RolesPlugin extends ZeppelinPluginClass { } } - let resultMessage = `Role added to ${assigned} ${assigned === 1 ? "member" : "members"}!`; + let resultMessage = `Added role **${role.name}** to ${assigned} ${assigned === 1 ? "member" : "members"}!`; if (alreadyHadRole) { resultMessage += ` ${alreadyHadRole} ${alreadyHadRole === 1 ? "member" : "members"} already had the role.`; } @@ -221,7 +232,10 @@ export class RolesPlugin extends ZeppelinPluginClass { mod: stripObjectToScalars(msg.author), }); - this.sendSuccessMessage(msg.channel, "Role removed from user!"); + this.sendSuccessMessage( + msg.channel, + `Removed role **${role.name}** removed from ${verboseUserMention(args.member.user)}!`, + ); } @d.command("massremoverole", " ") @@ -248,19 +262,30 @@ export class RolesPlugin extends ZeppelinPluginClass { if (!roleId) { return this.sendErrorMessage(msg.channel, "Invalid role id"); } - const role = this.guild.roles.get(roleId); const config = this.getConfigForMsg(msg); if (!config.assignable_roles.includes(roleId)) { return this.sendErrorMessage(msg.channel, "You cannot remove that role"); } + const role = this.guild.roles.get(roleId); + if (!role) { + this.logs.log(LogType.BOT_ALERT, { + body: `Unknown role configured for 'roles' plugin: ${roleId}`, + }); + return this.sendErrorMessage(msg.channel, "You cannot remove that role"); + } + const membersWithTheRole = members.filter(m => m.roles.includes(roleId)); let assigned = 0; const failed = []; const didNotHaveRole = members.length - membersWithTheRole.length; - msg.channel.createMessage(`Removing role from specified members...`); + msg.channel.createMessage( + `Removing role **${role.name}** from ${membersWithTheRole.length} ${ + membersWithTheRole.length === 1 ? "member" : "members" + }...`, + ); for (const member of membersWithTheRole) { try { @@ -278,7 +303,7 @@ export class RolesPlugin extends ZeppelinPluginClass { } } - let resultMessage = `Role removed from ${assigned} ${assigned === 1 ? "member" : "members"}!`; + let resultMessage = `Removed role **${role.name}** from ${assigned} ${assigned === 1 ? "member" : "members"}!`; if (didNotHaveRole) { resultMessage += ` ${didNotHaveRole} ${didNotHaveRole === 1 ? "member" : "members"} didn't have the role.`; } diff --git a/backend/src/plugins/Starboard.ts b/backend/src/plugins/Starboard.ts index 4302042a..db74fe64 100644 --- a/backend/src/plugins/Starboard.ts +++ b/backend/src/plugins/Starboard.ts @@ -26,6 +26,7 @@ const StarboardOpts = t.type({ channel_id: t.string, stars_required: t.number, star_emoji: tNullable(t.array(t.string)), + copy_full_embed: tNullable(t.boolean), enabled: tNullable(t.boolean), }); type TStarboardOpts = t.TypeOf; @@ -148,6 +149,7 @@ export class StarboardPlugin extends ZeppelinPluginClass { if (cfg.enabled == null) cfg.enabled = defaultStarboardOpts.enabled; if (cfg.star_emoji == null) cfg.star_emoji = defaultStarboardOpts.star_emoji; if (cfg.stars_required == null) cfg.stars_required = defaultStarboardOpts.stars_required; + if (cfg.copy_full_embed == null) cfg.copy_full_embed = false; }); return configs; @@ -224,7 +226,7 @@ export class StarboardPlugin extends ZeppelinPluginClass { const reactions = await this.starboardReactions.getAllReactionsForMessageId(msg.id); const reactionsCount = reactions.length; if (reactionsCount >= starboard.stars_required) { - await this.saveMessageToStarboard(msg, starboard.channel_id); + await this.saveMessageToStarboard(msg, starboard); } } } @@ -243,8 +245,8 @@ export class StarboardPlugin extends ZeppelinPluginClass { * Saves/posts a message to the specified starboard. * The message is posted as an embed and image attachments are included as the embed image. */ - async saveMessageToStarboard(msg: Message, starboardChannelId: string) { - const channel = this.guild.channels.get(starboardChannelId); + async saveMessageToStarboard(msg: Message, starboard: TStarboardOpts) { + const channel = this.guild.channels.get(starboard.channel_id); if (!channel) return; const time = moment(msg.timestamp, "x").format("YYYY-MM-DD [at] HH:mm:ss [UTC]"); @@ -256,6 +258,7 @@ export class StarboardPlugin extends ZeppelinPluginClass { author: { name: `${msg.author.username}#${msg.author.discriminator}`, }, + fields: [], timestamp: new Date(msg.timestamp).toISOString(), }; @@ -267,24 +270,35 @@ export class StarboardPlugin extends ZeppelinPluginClass { embed.description = msg.content; } - // Include attachments - if (msg.attachments.length) { - const attachment = msg.attachments[0]; - const ext = path - .extname(attachment.filename) - .slice(1) - .toLowerCase(); - if (["jpeg", "jpg", "png", "gif", "webp"].includes(ext)) { - embed.image = { url: attachment.url }; + // Merge media and - if copy_full_embed is enabled - fields and title from the first embed in the original message + if (msg.embeds.length > 0) { + if (msg.embeds[0].image) embed.image = msg.embeds[0].image; + + if (starboard.copy_full_embed) { + if (msg.embeds[0].title) { + const titleText = msg.embeds[0].url ? `[${msg.embeds[0].title}](${msg.embeds[0].url})` : msg.embeds[0].title; + embed.fields.push({ name: EMPTY_CHAR, value: titleText }); + } + + if (msg.embeds[0].fields) embed.fields.push(...msg.embeds[0].fields); } } - // Include any embed images in the original message - if (msg.embeds.length && msg.embeds[0].image) { - embed.image = msg.embeds[0].image; + // If there are no embeds, add the first image attachment explicitly + else if (msg.attachments.length) { + for (const attachment of msg.attachments) { + const ext = path + .extname(attachment.filename) + .slice(1) + .toLowerCase(); + if (!["jpeg", "jpg", "png", "gif", "webp"].includes(ext)) continue; + + embed.image = { url: attachment.url }; + break; + } } - embed.fields = [{ name: EMPTY_CHAR, value: `[Jump to message](${messageLink(msg)})` }]; + embed.fields.push({ name: EMPTY_CHAR, value: `[Jump to message](${messageLink(msg)})` }); const starboardMessage = await (channel as TextChannel).createMessage({ embed }); await this.starboardMessages.createStarboardMessage(channel.id, msg.id, starboardMessage.id); @@ -364,7 +378,7 @@ export class StarboardPlugin extends ZeppelinPluginClass { pin.id, ); if (existingStarboardMessage.length > 0) continue; - await this.saveMessageToStarboard(pin, starboardChannel.id); + await this.saveMessageToStarboard(pin, starboard); } this.sendSuccessMessage(msg.channel, `Pins migrated from <#${args.pinChannel.id}> to <#${starboardChannel.id}>!`); diff --git a/backend/src/plugins/Tags.ts b/backend/src/plugins/Tags.ts index b2284940..b4b4ff2f 100644 --- a/backend/src/plugins/Tags.ts +++ b/backend/src/plugins/Tags.ts @@ -133,7 +133,7 @@ export class TagsPlugin extends ZeppelinPluginClass { }, countdown(toDate) { - const target = this.parseDateTime(toDate); + const target = moment(this.parseDateTime(toDate)); const now = moment(); if (!target.isValid()) return ""; @@ -222,6 +222,10 @@ export class TagsPlugin extends ZeppelinPluginClass { return ""; }, }; + + for (const [name, fn] of Object.entries(this.tagFunctions)) { + this.tagFunctions[name] = (fn as any).bind(this.tagFunctions); + } } onUnload() { diff --git a/backend/src/plugins/UsernameSaver.ts b/backend/src/plugins/UsernameSaver.ts index fe6388b0..25cdcde4 100644 --- a/backend/src/plugins/UsernameSaver.ts +++ b/backend/src/plugins/UsernameSaver.ts @@ -1,15 +1,18 @@ import { decorators as d, GlobalPlugin } from "knub"; import { UsernameHistory } from "../data/UsernameHistory"; -import { Member, User } from "eris"; +import { Member, Message, User } from "eris"; import { GlobalZeppelinPlugin } from "./GlobalZeppelinPlugin"; +import { Queue } from "../Queue"; export class UsernameSaver extends GlobalZeppelinPlugin { public static pluginName = "username_saver"; protected usernameHistory: UsernameHistory; + protected updateQueue: Queue; async onLoad() { this.usernameHistory = new UsernameHistory(); + this.updateQueue = new Queue(); } protected async updateUsername(user: User) { @@ -21,13 +24,15 @@ export class UsernameSaver extends GlobalZeppelinPlugin { } } - @d.event("userUpdate", null, false) - async onUserUpdate(user: User) { - this.updateUsername(user); + @d.event("messageCreate", null) + async onMessage(msg: Message) { + if (msg.author.bot) return; + this.updateQueue.add(() => this.updateUsername(msg.author)); } - @d.event("guildMemberAdd", null, false) - async onGuildMemberAdd(_, member: Member) { - this.updateUsername(member.user); + @d.event("voiceChannelJoin", null) + async onVoiceChannelJoin(member: Member) { + if (member.user.bot) return; + this.updateQueue.add(() => this.updateUsername(member.user)); } } diff --git a/backend/src/plugins/Utility.ts b/backend/src/plugins/Utility.ts index 1cdcad6f..70df7b34 100644 --- a/backend/src/plugins/Utility.ts +++ b/backend/src/plugins/Utility.ts @@ -76,6 +76,7 @@ declare global { } import { Url, URL, URLSearchParams } from "url"; +import { Supporters } from "../data/Supporters"; const ConfigSchema = t.type({ can_roles: t.boolean, can_level: t.boolean, @@ -123,6 +124,17 @@ type MemberSearchParams = { "status-search"?: boolean; }; +type BanSearchParams = { + query?: string; + sort?: string; + "case-sensitive"?: boolean; + regex?: boolean; +}; + +enum SearchType { + MemberSearch, + BanSearch, +} class SearchError extends Error {} export class UtilityPlugin extends ZeppelinPluginClass { @@ -137,6 +149,7 @@ export class UtilityPlugin extends ZeppelinPluginClass { protected cases: GuildCases; protected savedMessages: GuildSavedMessages; protected archives: GuildArchives; + protected supporters: Supporters; protected lastFullMemberRefresh = 0; protected fullMemberRefreshPromise; @@ -199,6 +212,7 @@ export class UtilityPlugin extends ZeppelinPluginClass { this.cases = GuildCases.getGuildInstance(this.guildId); this.savedMessages = GuildSavedMessages.getGuildInstance(this.guildId); this.archives = GuildArchives.getGuildInstance(this.guildId); + this.supporters = new Supporters(); this.lastReload = Date.now(); @@ -342,6 +356,63 @@ export class UtilityPlugin extends ZeppelinPluginClass { msg.channel.createMessage(`The permission level of ${member.username}#${member.discriminator} is **${level}**`); } + protected async performBanSearch( + args: BanSearchParams, + page = 1, + perPage = SEARCH_RESULTS_PER_PAGE, + ): Promise<{ results: User[]; totalResults: number; page: number; lastPage: number; from: number; to: number }> { + let matchingBans = (await this.guild.getBans()).map(x => x.user); + + if (args.query) { + let queryRegex: RegExp; + if (args.regex) { + queryRegex = new RegExp(args.query.trimStart(), args["case-sensitive"] ? "" : "i"); + } else { + queryRegex = new RegExp(escapeStringRegexp(args.query.trimStart()), args["case-sensitive"] ? "" : "i"); + } + + if (!safeRegex(queryRegex)) { + throw new SearchError("Unsafe/too complex regex (star depth is limited to 1)"); + } + + matchingBans = matchingBans.filter(user => { + const fullUsername = `${user.username}#${user.discriminator}`; + if (fullUsername.match(queryRegex)) return true; + }); + } + + const [, sortDir, sortBy] = args.sort ? args.sort.match(/^(-?)(.*)$/) : [null, "ASC", "name"]; + const realSortDir = sortDir === "-" ? "DESC" : "ASC"; + + if (sortBy === "id") { + matchingBans.sort(sorter(m => BigInt(m.id), realSortDir)); + } else { + matchingBans.sort( + multiSorter([ + [m => m.username.toLowerCase(), realSortDir], + [m => m.discriminator, realSortDir], + ]), + ); + } + + const lastPage = Math.max(1, Math.ceil(matchingBans.length / perPage)); + page = Math.min(lastPage, Math.max(1, page)); + + const from = (page - 1) * perPage; + const to = Math.min(from + perPage, matchingBans.length); + + const pageMembers = matchingBans.slice(from, to); + + return { + results: pageMembers, + totalResults: matchingBans.length, + page, + lastPage, + from: from + 1, + to, + }; + } + protected async performMemberSearch( args: MemberSearchParams, page = 1, @@ -457,6 +528,26 @@ export class UtilityPlugin extends ZeppelinPluginClass { }; } + protected formatSearchResultList(members: Array): string { + const longestId = members.reduce((longest, member) => Math.max(longest, member.id.length), 0); + const lines = members.map(member => { + const paddedId = member.id.padEnd(longestId, " "); + let line; + if (member instanceof Member) { + line = `${paddedId} ${member.user.username}#${member.user.discriminator}`; + if (member.nick) line += ` (${member.nick})`; + } else { + line = `${paddedId} ${member.username}#${member.discriminator}`; + } + return line; + }); + return lines.join("\n"); + } + + protected formatSearchResultIdList(members: Array): string { + return members.map(m => m.id).join(" "); + } + @d.command("search", "[query:string$]", { aliases: ["s"], options: [ @@ -542,56 +633,153 @@ export class UtilityPlugin extends ZeppelinPluginClass { "status-search"?: boolean; }, ) { - const formatSearchResultList = (members: Member[]): string => { - const longestId = members.reduce((longest, member) => Math.max(longest, member.id.length), 0); - const lines = members.map(member => { - const paddedId = member.id.padEnd(longestId, " "); - let line = `${paddedId} ${member.user.username}#${member.user.discriminator}`; - if (member.nick) line += ` (${member.nick})`; - return line; - }); - return lines.join("\n"); - }; - - const formatSearchResultIdList = (members: Member[]): string => { - return members.map(m => m.id).join(" "); - }; - // If we're exporting the results, we don't need all the fancy schmancy pagination stuff. // Just get the results and dump them in an archive. if (args.export) { - let results; - try { - results = await this.performMemberSearch(args, 1, SEARCH_EXPORT_LIMIT); - } catch (e) { - if (e instanceof SearchError) { - return this.sendErrorMessage(msg.channel, e.message); - } + return this.archiveSearch(args, SearchType.MemberSearch, msg); + } else { + return this.displaySearch(args, SearchType.MemberSearch, msg); + } + } - throw e; + @d.command("bansearch", "[query:string$]", { + aliases: ["bs"], + options: [ + { + name: "page", + shortcut: "p", + type: "number", + }, + { + name: "sort", + type: "string", + }, + { + name: "case-sensitive", + shortcut: "cs", + isSwitch: true, + }, + { + name: "export", + shortcut: "e", + isSwitch: true, + }, + { + name: "ids", + isSwitch: true, + }, + { + name: "regex", + shortcut: "re", + isSwitch: true, + }, + ], + extra: { + info: { + description: "Search banned users", + basicUsage: "!bansearch dragory", + optionDescriptions: { + sort: + "Change how the results are sorted. Possible values are 'id' and 'name'. Prefix with a dash, e.g. '-id', to reverse sorting.", + "case-sensitive": "By default, the search is case-insensitive. Use this to make it case-sensitive instead.", + export: "If set, the full search results are exported as an archive", + }, + }, + }, + }) + @d.permission("can_search") + async banSearchCmd( + msg: Message, + args: { + query?: string; + page?: number; + sort?: string; + "case-sensitive"?: boolean; + export?: boolean; + ids?: boolean; + regex?: boolean; + }, + ) { + if (args.export) { + return this.archiveSearch(args, SearchType.BanSearch, msg); + } else { + return this.displaySearch(args, SearchType.BanSearch, msg); + } + } + + async cleanMessages(channel: Channel, savedMessages: SavedMessage[], mod: User) { + this.logs.ignoreLog(LogType.MESSAGE_DELETE, savedMessages[0].id); + this.logs.ignoreLog(LogType.MESSAGE_DELETE_BULK, savedMessages[0].id); + + // Delete & archive in ID order + savedMessages = Array.from(savedMessages).sort((a, b) => (a.id > b.id ? 1 : -1)); + const idsToDelete = savedMessages.map(m => m.id); + + // Make sure the deletions aren't double logged + idsToDelete.forEach(id => this.logs.ignoreLog(LogType.MESSAGE_DELETE, id)); + this.logs.ignoreLog(LogType.MESSAGE_DELETE_BULK, idsToDelete[0]); + + // Actually delete the messages + await this.bot.deleteMessages(channel.id, idsToDelete); + await this.savedMessages.markBulkAsDeleted(idsToDelete); + + // Create an archive + const archiveId = await this.archives.createFromSavedMessages(savedMessages, this.guild); + const archiveUrl = this.archives.getUrl(this.knub.getGlobalConfig().url, archiveId); + + this.logs.log(LogType.CLEAN, { + mod: stripObjectToScalars(mod), + channel: stripObjectToScalars(channel), + count: savedMessages.length, + archiveUrl, + }); + + return { archiveUrl }; + } + + async archiveSearch(args: any, searchType: SearchType, msg: Message) { + let results; + try { + switch (searchType) { + case SearchType.MemberSearch: + results = await this.performMemberSearch(args, 1, SEARCH_EXPORT_LIMIT); + break; + case SearchType.BanSearch: + results = await this.performBanSearch(args, 1, SEARCH_EXPORT_LIMIT); + break; + } + } catch (e) { + if (e instanceof SearchError) { + return this.sendErrorMessage(msg.channel, e.message); } - if (results.totalResults === 0) { - return this.sendErrorMessage(msg.channel, "No results found"); - } + throw e; + } - const resultList = args.ids ? formatSearchResultIdList(results.results) : formatSearchResultList(results.results); + if (results.totalResults === 0) { + return this.sendErrorMessage(msg.channel, "No results found"); + } - const archiveId = await this.archives.create( - trimLines(` + const resultList = args.ids + ? this.formatSearchResultIdList(results.results) + : this.formatSearchResultList(results.results); + + const archiveId = await this.archives.create( + trimLines(` Search results (total ${results.totalResults}): ${resultList} `), - moment().add(1, "hour"), - ); - const url = await this.archives.getUrl(this.knub.getGlobalConfig().url, archiveId); + moment().add(1, "hour"), + ); + const url = await this.archives.getUrl(this.knub.getGlobalConfig().url, archiveId); - msg.channel.createMessage(`Exported search results: ${url}`); + msg.channel.createMessage(`Exported search results: ${url}`); - return; - } + return; + } + async displaySearch(args: any, searchType: SearchType, msg: Message) { // If we're not exporting, load 1 page of search results at a time and allow the user to switch pages with reactions let originalSearchMsg: Message = null; let searching = false; @@ -618,7 +806,14 @@ export class UtilityPlugin extends ZeppelinPluginClass { let searchResult; try { - searchResult = await this.performMemberSearch(args, page, perPage); + switch (searchType) { + case SearchType.MemberSearch: + searchResult = await this.performMemberSearch(args, page, perPage); + break; + case SearchType.BanSearch: + searchResult = await this.performBanSearch(args, page, perPage); + break; + } } catch (e) { if (e instanceof SearchError) { return this.sendErrorMessage(msg.channel, e.message); @@ -640,8 +835,8 @@ export class UtilityPlugin extends ZeppelinPluginClass { : `Found ${searchResult.totalResults} ${resultWord}`; const resultList = args.ids - ? formatSearchResultIdList(searchResult.results) - : formatSearchResultList(searchResult.results); + ? this.formatSearchResultIdList(searchResult.results) + : this.formatSearchResultList(searchResult.results); const result = trimLines(` ${headerText} @@ -694,36 +889,6 @@ export class UtilityPlugin extends ZeppelinPluginClass { loadSearchPage(currentPage); } - async cleanMessages(channel: Channel, savedMessages: SavedMessage[], mod: User) { - this.logs.ignoreLog(LogType.MESSAGE_DELETE, savedMessages[0].id); - this.logs.ignoreLog(LogType.MESSAGE_DELETE_BULK, savedMessages[0].id); - - // Delete & archive in ID order - savedMessages = Array.from(savedMessages).sort((a, b) => (a.id > b.id ? 1 : -1)); - const idsToDelete = savedMessages.map(m => m.id); - - // Make sure the deletions aren't double logged - idsToDelete.forEach(id => this.logs.ignoreLog(LogType.MESSAGE_DELETE, id)); - this.logs.ignoreLog(LogType.MESSAGE_DELETE_BULK, idsToDelete[0]); - - // Actually delete the messages - await this.bot.deleteMessages(channel.id, idsToDelete); - await this.savedMessages.markBulkAsDeleted(idsToDelete); - - // Create an archive - const archiveId = await this.archives.createFromSavedMessages(savedMessages, this.guild); - const archiveUrl = this.archives.getUrl(this.knub.getGlobalConfig().url, archiveId); - - this.logs.log(LogType.CLEAN, { - mod: stripObjectToScalars(mod), - channel: stripObjectToScalars(channel), - count: savedMessages.length, - archiveUrl, - }); - - return { archiveUrl }; - } - @d.command("clean", "", { options: [ { @@ -1069,8 +1234,6 @@ export class UtilityPlugin extends ZeppelinPluginClass { }) @d.permission("can_server") async serverCmd(msg: Message) { - await this.refreshMembersIfNeeded(); - const embed: EmbedOptions = { fields: [], color: parseInt("6b80cf", 16), @@ -1121,10 +1284,6 @@ export class UtilityPlugin extends ZeppelinPluginClass { : this.guild.members.filter(m => m.status !== "offline").length; const offlineMemberCount = this.guild.memberCount - onlineMemberCount; - const onlineStatusMemberCount = this.guild.members.filter(m => m.status === "online").length; - const dndStatusMemberCount = this.guild.members.filter(m => m.status === "dnd").length; - const idleStatusMemberCount = this.guild.members.filter(m => m.status === "idle").length; - let memberCountTotalLines = `Total: **${formatNumber(totalMembers)}**`; if (restGuild.maxMembers) { memberCountTotalLines += `\nMax: **${formatNumber(restGuild.maxMembers)}**`; @@ -1142,9 +1301,6 @@ export class UtilityPlugin extends ZeppelinPluginClass { ${memberCountTotalLines} ${memberCountOnlineLines} Offline: **${formatNumber(offlineMemberCount)}** - <:zep_online:665907874450636810> Online: **${formatNumber(onlineStatusMemberCount)}** - <:zep_idle:665908128331726848> Idle: **${formatNumber(idleStatusMemberCount)}** - <:zep_dnd:665908138741858365> DND: **${formatNumber(dndStatusMemberCount)}** `), }); @@ -1495,38 +1651,41 @@ export class UtilityPlugin extends ZeppelinPluginClass { const loadedPlugins = Array.from(this.knub.getGuildData(this.guildId).loadedPlugins.keys()); loadedPlugins.sort(); - const supporters = [ - ["Flokie", 10], - ["CmdData", 1], - ["JackDaniel", 1], - ]; - supporters.sort(sorter(r => r[1], "DESC")); - const aboutContent: MessageContent = { embed: { title: `About ${this.bot.user.username}`, fields: [ { name: "Status", - value: - basicInfoRows - .map(([label, value]) => { - return `${label}: **${value}**`; - }) - .join("\n") + embedPadding, + value: basicInfoRows + .map(([label, value]) => { + return `${label}: **${value}**`; + }) + .join("\n"), }, { name: `Loaded plugins on this server (${loadedPlugins.length})`, value: loadedPlugins.join(", "), }, - { - name: "Zeppelin supporters 🎉", - value: supporters.map(s => `**${s[0]}** ${s[1]}€/mo`).join("\n"), - }, ], }, }; + const supporters = await this.supporters.getAll(); + supporters.sort( + multiSorter([ + [r => r.amount, "DESC"], + [r => r.name.toLowerCase(), "ASC"], + ]), + ); + + if (supporters.length) { + aboutContent.embed.fields.push({ + name: "Zeppelin supporters 🎉", + value: supporters.map(s => `**${s.name}** ${s.amount ? `${s.amount}€/mo` : ""}`.trim()).join("\n"), + }); + } + // For the embed color, find the highest colored role the bot has - this is their color on the server as well const botMember = await resolveMember(this.bot, this.guild, this.bot.user.id); let botRoles = botMember.roles.map(r => (msg.channel as GuildChannel).guild.roles.get(r)); diff --git a/backend/src/plugins/ZeppelinPluginClass.ts b/backend/src/plugins/ZeppelinPluginClass.ts index c6447173..a8f8b483 100644 --- a/backend/src/plugins/ZeppelinPluginClass.ts +++ b/backend/src/plugins/ZeppelinPluginClass.ts @@ -118,6 +118,12 @@ export class ZeppelinPluginClass { + if (options == null) { + options = { + enabled: false, + }; + } + const defaultOptions: any = this.getStaticDefaultOptions(); let mergedConfig = configUtils.mergeConfig({}, defaultOptions.config || {}, options.config || {}); const mergedOverrides = options.replaceDefaultOverrides