3
0
Fork 0
mirror of https://github.com/ZeppelinBot/Zeppelin.git synced 2025-03-16 22:21:51 +00:00

Message cleanup tweaks

This commit is contained in:
Dragory 2020-06-01 21:28:07 +03:00
parent 97d144e9b4
commit 80f6f69ccd
No known key found for this signature in database
GPG key ID: 5F387BA66DF8AAC1

View file

@ -1,4 +1,4 @@
import { Brackets, getRepository, Repository } from "typeorm"; import { getRepository, In, Repository } from "typeorm";
import { BaseGuildRepository } from "./BaseGuildRepository"; import { BaseGuildRepository } from "./BaseGuildRepository";
import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage"; import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage";
import { QueuedEventEmitter } from "../QueuedEventEmitter"; import { QueuedEventEmitter } from "../QueuedEventEmitter";
@ -9,7 +9,6 @@ import { isAPI } from "../globals";
import { connection } from "./db"; import { connection } from "./db";
const CLEANUP_INTERVAL = 5 * MINUTES; const CLEANUP_INTERVAL = 5 * MINUTES;
let cleanupPromise = Promise.resolve();
/** /**
* How long message edits, deletions, etc. will include the original message content. * How long message edits, deletions, etc. will include the original message content.
@ -17,8 +16,11 @@ let cleanupPromise = Promise.resolve();
*/ */
const RETENTION_PERIOD = 1 * DAYS; const RETENTION_PERIOD = 1 * DAYS;
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES; const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
const CLEAN_PER_LOOP = 250;
async function cleanup() { async function cleanup() {
const repo = getRepository(SavedMessage);
const deletedAtThreshold = moment() const deletedAtThreshold = moment()
.subtract(CLEANUP_INTERVAL, "ms") .subtract(CLEANUP_INTERVAL, "ms")
.format(DBDateFormat); .format(DBDateFormat);
@ -29,28 +31,39 @@ async function cleanup() {
.subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms") .subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms")
.format(DBDateFormat); .format(DBDateFormat);
const query = ` // SELECT + DELETE messages in batches
DELETE FROM messages // This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
WHERE ( // when a message was being inserted at the same time
deleted_at IS NOT NULL let rows;
AND deleted_at <= ? do {
) rows = await connection.query(
OR ( `
posted_at <= ? SELECT id
AND is_permanent = 0 FROM messages
) WHERE (
OR ( deleted_at IS NOT NULL
is_bot = 1 AND deleted_at <= ?
AND posted_at <= ? )
AND is_permanent = 0 OR (
) posted_at <= ?
LIMIT ${25_000} AND is_permanent = 0
`; )
OR (
is_bot = 1
AND posted_at <= ?
AND is_permanent = 0
)
LIMIT ${CLEAN_PER_LOOP}
`,
[deletedAtThreshold, postedAtThreshold, botPostedAtThreshold],
);
cleanupPromise = (async () => { if (rows.length > 0) {
await connection.query(query, [deletedAtThreshold, postedAtThreshold, botPostedAtThreshold]); await repo.delete({
})(); id: In(rows.map(r => r.id)),
await cleanupPromise; });
}
} while (rows.length === CLEAN_PER_LOOP);
setTimeout(cleanup, CLEANUP_INTERVAL); setTimeout(cleanup, CLEANUP_INTERVAL);
} }
@ -139,8 +152,8 @@ export class GuildSavedMessages extends BaseGuildRepository {
let query = this.messages let query = this.messages
.createQueryBuilder() .createQueryBuilder()
.where("guild_id = :guild_id", { guild_id: this.guildId }) .where("guild_id = :guild_id", { guild_id: this.guildId })
.andWhere("user_id = :user_id", { user_id: userId })
.andWhere("channel_id = :channel_id", { channel_id: channelId }) .andWhere("channel_id = :channel_id", { channel_id: channelId })
.andWhere("user_id = :user_id", { user_id: userId })
.andWhere("id > :afterId", { afterId }) .andWhere("id > :afterId", { afterId })
.andWhere("deleted_at IS NULL"); .andWhere("deleted_at IS NULL");
@ -166,8 +179,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
this.toBePermanent.delete(data.id); this.toBePermanent.delete(data.id);
} }
await cleanupPromise;
try { try {
await this.messages.insert(data); await this.messages.insert(data);
} catch (e) { } catch (e) {
@ -184,8 +195,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
const existingSavedMsg = await this.find(msg.id); const existingSavedMsg = await this.find(msg.id);
if (existingSavedMsg) return; if (existingSavedMsg) return;
await cleanupPromise;
const savedMessageData = this.msgToSavedMessageData(msg); const savedMessageData = this.msgToSavedMessageData(msg);
const postedAt = moment.utc(msg.timestamp, "x").format("YYYY-MM-DD HH:mm:ss.SSS"); const postedAt = moment.utc(msg.timestamp, "x").format("YYYY-MM-DD HH:mm:ss.SSS");
@ -203,8 +212,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
} }
async markAsDeleted(id) { async markAsDeleted(id) {
await cleanupPromise;
await this.messages await this.messages
.createQueryBuilder("messages") .createQueryBuilder("messages")
.update() .update()
@ -228,8 +235,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
* If any messages were marked as deleted, also emits the deleteBulk event. * If any messages were marked as deleted, also emits the deleteBulk event.
*/ */
async markBulkAsDeleted(ids) { async markBulkAsDeleted(ids) {
await cleanupPromise;
const deletedAt = moment().format("YYYY-MM-DD HH:mm:ss.SSS"); const deletedAt = moment().format("YYYY-MM-DD HH:mm:ss.SSS");
await this.messages await this.messages
@ -256,8 +261,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
const oldMessage = await this.messages.findOne(id); const oldMessage = await this.messages.findOne(id);
if (!oldMessage) return; if (!oldMessage) return;
await cleanupPromise;
const newMessage = { ...oldMessage, data: newData }; const newMessage = { ...oldMessage, data: newData };
await this.messages.update( await this.messages.update(
@ -279,7 +282,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
async setPermanent(id: string) { async setPermanent(id: string) {
const savedMsg = await this.find(id); const savedMsg = await this.find(id);
if (savedMsg) { if (savedMsg) {
await cleanupPromise;
await this.messages.update( await this.messages.update(
{ id }, { id },
{ {