diff --git a/backend/src/data/GuildNicknameHistory.ts b/backend/src/data/GuildNicknameHistory.ts index 58ed7273..91d9d533 100644 --- a/backend/src/data/GuildNicknameHistory.ts +++ b/backend/src/data/GuildNicknameHistory.ts @@ -1,7 +1,22 @@ import { BaseGuildRepository } from "./BaseGuildRepository"; -import { getRepository, Repository } from "typeorm"; +import { getRepository, In, Repository } from "typeorm"; import { NicknameHistoryEntry } from "./entities/NicknameHistoryEntry"; -import { sorter } from "../utils"; +import { MINUTES, SECONDS, sorter } from "../utils"; +import { MAX_USERNAME_ENTRIES_PER_USER } from "./UsernameHistory"; +import { isAPI } from "../globals"; +import { cleanupNicknames } from "./cleanup/nicknames"; + +if (!isAPI()) { + const CLEANUP_INTERVAL = 5 * MINUTES; + + async function cleanup() { + await cleanupNicknames(); + setTimeout(cleanup, CLEANUP_INTERVAL); + } + + // Start first cleanup 30 seconds after startup + setTimeout(cleanup, 30 * SECONDS); +} export const MAX_NICKNAME_ENTRIES_PER_USER = 10; @@ -44,25 +59,20 @@ export class GuildNicknameHistory extends BaseGuildRepository { nickname, }); - // Cleanup (leave only the last MAX_NICKNAME_ENTRIES_PER_USER entries) - const lastEntries = await this.getByUserId(userId); - if (lastEntries.length > MAX_NICKNAME_ENTRIES_PER_USER) { - const earliestEntry = lastEntries - .sort(sorter("timestamp", "DESC")) - .slice(0, 10) - .reduce((earliest, entry) => { - if (earliest == null) return entry; - if (entry.id < earliest.id) return entry; - return earliest; - }, null); + // Cleanup (leave only the last MAX_USERNAME_ENTRIES_PER_USER entries) + const toDelete = await this.nicknameHistory + .createQueryBuilder() + .where("guild_id = :guildId", { guildId: this.guildId }) + .andWhere("user_id = :userId", { userId }) + .orderBy("id", "DESC") + .skip(MAX_USERNAME_ENTRIES_PER_USER) + .take(99_999) + .getMany(); - this.nicknameHistory - .createQueryBuilder() - .where("guild_id = :guildId", { guildId: this.guildId }) - .andWhere("user_id = :userId", { userId }) - .andWhere("id < :id", { id: earliestEntry.id }) - .delete() - .execute(); + if (toDelete.length > 0) { + await this.nicknameHistory.delete({ + id: In(toDelete.map(v => v.id)), + }); } } } diff --git a/backend/src/data/GuildSavedMessages.ts b/backend/src/data/GuildSavedMessages.ts index fcc6cb02..a052ef66 100644 --- a/backend/src/data/GuildSavedMessages.ts +++ b/backend/src/data/GuildSavedMessages.ts @@ -7,68 +7,16 @@ import moment from "moment-timezone"; import { DAYS, DBDateFormat, MINUTES, SECONDS } from "../utils"; import { isAPI } from "../globals"; import { connection } from "./db"; - -const CLEANUP_INTERVAL = 5 * MINUTES; - -/** - * How long message edits, deletions, etc. will include the original message content. - * This is very heavy storage-wise, so keeping it as low as possible is ideal. - */ -const RETENTION_PERIOD = 1 * DAYS; -const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES; -const CLEAN_PER_LOOP = 250; - -async function cleanup() { - const repo = getRepository(SavedMessage); - - const deletedAtThreshold = moment() - .subtract(CLEANUP_INTERVAL, "ms") - .format(DBDateFormat); - const postedAtThreshold = moment() - .subtract(RETENTION_PERIOD, "ms") - .format(DBDateFormat); - const botPostedAtThreshold = moment() - .subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms") - .format(DBDateFormat); - - // SELECT + DELETE messages in batches - // This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below - // when a message was being inserted at the same time - let rows; - do { - rows = await connection.query( - ` - SELECT id - FROM messages - WHERE ( - deleted_at IS NOT NULL - AND deleted_at <= ? - ) - OR ( - posted_at <= ? - AND is_permanent = 0 - ) - OR ( - is_bot = 1 - AND posted_at <= ? - AND is_permanent = 0 - ) - LIMIT ${CLEAN_PER_LOOP} - `, - [deletedAtThreshold, postedAtThreshold, botPostedAtThreshold], - ); - - if (rows.length > 0) { - await repo.delete({ - id: In(rows.map(r => r.id)), - }); - } - } while (rows.length === CLEAN_PER_LOOP); - - setTimeout(cleanup, CLEANUP_INTERVAL); -} +import { cleanupMessages } from "./cleanup/messages"; if (!isAPI()) { + const CLEANUP_INTERVAL = 5 * MINUTES; + + async function cleanup() { + await cleanupMessages(); + setTimeout(cleanup, CLEANUP_INTERVAL); + } + // Start first cleanup 30 seconds after startup setTimeout(cleanup, 30 * SECONDS); } diff --git a/backend/src/data/UsernameHistory.ts b/backend/src/data/UsernameHistory.ts index e97d58cb..addbf823 100644 --- a/backend/src/data/UsernameHistory.ts +++ b/backend/src/data/UsernameHistory.ts @@ -1,9 +1,24 @@ -import { getRepository, Repository } from "typeorm"; +import { getRepository, In, Repository } from "typeorm"; import { UsernameHistoryEntry } from "./entities/UsernameHistoryEntry"; -import { sorter } from "../utils"; +import { MINUTES, SECONDS, sorter } from "../utils"; import { BaseRepository } from "./BaseRepository"; +import { connection } from "./db"; +import { isAPI } from "../globals"; +import { cleanupUsernames } from "./cleanup/usernames"; -export const MAX_USERNAME_ENTRIES_PER_USER = 10; +if (!isAPI()) { + const CLEANUP_INTERVAL = 5 * MINUTES; + + async function cleanup() { + await cleanupUsernames(); + setTimeout(cleanup, CLEANUP_INTERVAL); + } + + // Start first cleanup 30 seconds after startup + setTimeout(cleanup, 1 * SECONDS); +} + +export const MAX_USERNAME_ENTRIES_PER_USER = 5; export class UsernameHistory extends BaseRepository { private usernameHistory: Repository; @@ -43,23 +58,18 @@ export class UsernameHistory extends BaseRepository { }); // Cleanup (leave only the last MAX_USERNAME_ENTRIES_PER_USER entries) - const lastEntries = await this.getByUserId(userId); - if (lastEntries.length > MAX_USERNAME_ENTRIES_PER_USER) { - const earliestEntry = lastEntries - .sort(sorter("timestamp", "DESC")) - .slice(0, 10) - .reduce((earliest, entry) => { - if (earliest == null) return entry; - if (entry.id < earliest.id) return entry; - return earliest; - }, null); + const toDelete = await this.usernameHistory + .createQueryBuilder() + .where("user_id = :userId", { userId }) + .orderBy("id", "DESC") + .skip(MAX_USERNAME_ENTRIES_PER_USER) + .take(99_999) + .getMany(); - this.usernameHistory - .createQueryBuilder() - .andWhere("user_id = :userId", { userId }) - .andWhere("id < :id", { id: earliestEntry.id }) - .delete() - .execute(); + if (toDelete.length > 0) { + await this.usernameHistory.delete({ + id: In(toDelete.map(v => v.id)), + }); } } } diff --git a/backend/src/data/cleanup/messages.ts b/backend/src/data/cleanup/messages.ts new file mode 100644 index 00000000..626a062e --- /dev/null +++ b/backend/src/data/cleanup/messages.ts @@ -0,0 +1,68 @@ +import { DAYS, DBDateFormat, MINUTES } from "../../utils"; +import { getRepository, In } from "typeorm"; +import { SavedMessage } from "../entities/SavedMessage"; +import moment from "moment-timezone"; +import { connection } from "../db"; + +/** + * How long message edits, deletions, etc. will include the original message content. + * This is very heavy storage-wise, so keeping it as low as possible is ideal. + */ +const RETENTION_PERIOD = 1 * DAYS; +const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES; +const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES; +const CLEAN_PER_LOOP = 250; + +export async function cleanupMessages(): Promise { + let cleaned = 0; + + const messagesRepository = getRepository(SavedMessage); + + const deletedAtThreshold = moment() + .subtract(DELETED_MESSAGE_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + const postedAtThreshold = moment() + .subtract(RETENTION_PERIOD, "ms") + .format(DBDateFormat); + const botPostedAtThreshold = moment() + .subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + + // SELECT + DELETE messages in batches + // This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below + // when a message was being inserted at the same time + let rows; + do { + rows = await connection.query( + ` + SELECT id + FROM messages + WHERE ( + deleted_at IS NOT NULL + AND deleted_at <= ? + ) + OR ( + posted_at <= ? + AND is_permanent = 0 + ) + OR ( + is_bot = 1 + AND posted_at <= ? + AND is_permanent = 0 + ) + LIMIT ${CLEAN_PER_LOOP} + `, + [deletedAtThreshold, postedAtThreshold, botPostedAtThreshold], + ); + + if (rows.length > 0) { + await messagesRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + return cleaned; +} diff --git a/backend/src/data/cleanup/nicknames.ts b/backend/src/data/cleanup/nicknames.ts new file mode 100644 index 00000000..3f41084d --- /dev/null +++ b/backend/src/data/cleanup/nicknames.ts @@ -0,0 +1,41 @@ +import { getRepository, In } from "typeorm"; +import moment from "moment-timezone"; +import { NicknameHistoryEntry } from "../entities/NicknameHistoryEntry"; +import { DAYS, DBDateFormat } from "../../utils"; +import { connection } from "../db"; + +export const NICKNAME_RETENTION_PERIOD = 30 * DAYS; +const CLEAN_PER_LOOP = 500; + +export async function cleanupNicknames(): Promise { + let cleaned = 0; + + const nicknameHistoryRepository = getRepository(NicknameHistoryEntry); + const dateThreshold = moment() + .subtract(NICKNAME_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + + // Clean old nicknames (NICKNAME_RETENTION_PERIOD) + let rows; + do { + rows = await connection.query( + ` + SELECT id + FROM nickname_history + WHERE timestamp < ? + LIMIT ${CLEAN_PER_LOOP} + `, + [dateThreshold], + ); + + if (rows.length > 0) { + await nicknameHistoryRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + return cleaned; +} diff --git a/backend/src/data/cleanup/usernames.ts b/backend/src/data/cleanup/usernames.ts new file mode 100644 index 00000000..3e385b25 --- /dev/null +++ b/backend/src/data/cleanup/usernames.ts @@ -0,0 +1,45 @@ +import { getRepository, In } from "typeorm"; +import moment from "moment-timezone"; +import { UsernameHistoryEntry } from "../entities/UsernameHistoryEntry"; +import { DAYS, DBDateFormat } from "../../utils"; +import { connection } from "../db"; + +export const USERNAME_RETENTION_PERIOD = 30 * DAYS; +const CLEAN_PER_LOOP = 500; + +export async function cleanupUsernames(): Promise { + let cleaned = 0; + + const usernameHistoryRepository = getRepository(UsernameHistoryEntry); + const dateThreshold = moment() + .subtract(USERNAME_RETENTION_PERIOD, "ms") + .format(DBDateFormat); + + // Clean old usernames (USERNAME_RETENTION_PERIOD) + let rows; + do { + rows = await connection.query( + ` + SELECT id + FROM username_history + WHERE timestamp < ? + LIMIT ${CLEAN_PER_LOOP} + `, + [dateThreshold], + ); + + if (rows.length > 0) { + console.log( + "ids", + rows.map(r => r.id), + ); + await usernameHistoryRepository.delete({ + id: In(rows.map(r => r.id)), + }); + } + + cleaned += rows.length; + } while (rows.length === CLEAN_PER_LOOP); + + return cleaned; +} diff --git a/backend/src/plugins/NameHistory.ts b/backend/src/plugins/NameHistory.ts index 19ce73f2..031841e7 100644 --- a/backend/src/plugins/NameHistory.ts +++ b/backend/src/plugins/NameHistory.ts @@ -1,10 +1,13 @@ import { decorators as d, IPluginOptions } from "knub"; import { GuildNicknameHistory, MAX_NICKNAME_ENTRIES_PER_USER } from "../data/GuildNicknameHistory"; import { Member, Message } from "eris"; -import { createChunkedMessage, disableCodeBlocks } from "../utils"; +import { createChunkedMessage, DAYS, disableCodeBlocks } from "../utils"; import { ZeppelinPlugin } from "./ZeppelinPlugin"; import { MAX_USERNAME_ENTRIES_PER_USER, UsernameHistory } from "../data/UsernameHistory"; import * as t from "io-ts"; +import { NICKNAME_RETENTION_PERIOD } from "../data/cleanup/nicknames"; +import moment from "moment-timezone"; +import { USERNAME_RETENTION_PERIOD } from "../data/cleanup/usernames"; const ConfigSchema = t.type({ can_view: t.boolean, @@ -59,23 +62,38 @@ export class NameHistoryPlugin extends ZeppelinPlugin { const user = this.bot.users.get(args.userId); const currentUsername = user ? `${user.username}#${user.discriminator}` : args.userId; + const nicknameDays = Math.round(NICKNAME_RETENTION_PERIOD / DAYS); + const usernameDays = Math.round(USERNAME_RETENTION_PERIOD / DAYS); + let message = `Name history for **${currentUsername}**:`; if (nicknameRows.length) { - message += `\n\n__Last ${MAX_NICKNAME_ENTRIES_PER_USER} nicknames:__\n${nicknameRows.join("\n")}`; + message += `\n\n__Last ${MAX_NICKNAME_ENTRIES_PER_USER} nicknames within ${nicknameDays} days:__\n${nicknameRows.join( + "\n", + )}`; } if (usernameRows.length) { - message += `\n\n__Last ${MAX_USERNAME_ENTRIES_PER_USER} usernames:__\n${usernameRows.join("\n")}`; + message += `\n\n__Last ${MAX_USERNAME_ENTRIES_PER_USER} usernames within ${usernameDays} days:__\n${usernameRows.join( + "\n", + )}`; } createChunkedMessage(msg.channel, message); } - @d.event("guildMemberUpdate") - async onGuildMemberUpdate(_, member: Member) { + async updateNickname(member: Member) { const latestEntry = await this.nicknameHistory.getLastEntry(member.id); if (!latestEntry || latestEntry.nickname !== member.nick) { - // tslint:disable-line await this.nicknameHistory.addEntry(member.id, member.nick); } } + + @d.event("messageCreate") + async onMessage(msg: Message) { + this.updateNickname(msg.member); + } + + @d.event("voiceChannelJoin") + async onVoiceChannelJoin(member: Member) { + this.updateNickname(member); + } } diff --git a/backend/src/plugins/UsernameSaver.ts b/backend/src/plugins/UsernameSaver.ts index fe6388b0..140627d6 100644 --- a/backend/src/plugins/UsernameSaver.ts +++ b/backend/src/plugins/UsernameSaver.ts @@ -1,6 +1,6 @@ import { decorators as d, GlobalPlugin } from "knub"; import { UsernameHistory } from "../data/UsernameHistory"; -import { Member, User } from "eris"; +import { Member, Message, User } from "eris"; import { GlobalZeppelinPlugin } from "./GlobalZeppelinPlugin"; export class UsernameSaver extends GlobalZeppelinPlugin { @@ -21,13 +21,15 @@ export class UsernameSaver extends GlobalZeppelinPlugin { } } - @d.event("userUpdate", null, false) - async onUserUpdate(user: User) { - this.updateUsername(user); + @d.event("messageCreate") + async onMessage(msg: Message) { + if (msg.author.bot) return; + this.updateUsername(msg.author); } - @d.event("guildMemberAdd", null, false) - async onGuildMemberAdd(_, member: Member) { + @d.event("voiceChannelJoin") + async onVoiceChannelJoin(member: Member) { + if (member.user.bot) return; this.updateUsername(member.user); } }