mirror of
https://github.com/ZeppelinBot/Zeppelin.git
synced 2025-03-16 22:21:51 +00:00
Message cleanup tweaks
This commit is contained in:
parent
97d144e9b4
commit
80f6f69ccd
1 changed files with 37 additions and 35 deletions
|
@ -1,4 +1,4 @@
|
||||||
import { Brackets, getRepository, Repository } from "typeorm";
|
import { getRepository, In, Repository } from "typeorm";
|
||||||
import { BaseGuildRepository } from "./BaseGuildRepository";
|
import { BaseGuildRepository } from "./BaseGuildRepository";
|
||||||
import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage";
|
import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage";
|
||||||
import { QueuedEventEmitter } from "../QueuedEventEmitter";
|
import { QueuedEventEmitter } from "../QueuedEventEmitter";
|
||||||
|
@ -9,7 +9,6 @@ import { isAPI } from "../globals";
|
||||||
import { connection } from "./db";
|
import { connection } from "./db";
|
||||||
|
|
||||||
const CLEANUP_INTERVAL = 5 * MINUTES;
|
const CLEANUP_INTERVAL = 5 * MINUTES;
|
||||||
let cleanupPromise = Promise.resolve();
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* How long message edits, deletions, etc. will include the original message content.
|
* How long message edits, deletions, etc. will include the original message content.
|
||||||
|
@ -17,8 +16,11 @@ let cleanupPromise = Promise.resolve();
|
||||||
*/
|
*/
|
||||||
const RETENTION_PERIOD = 1 * DAYS;
|
const RETENTION_PERIOD = 1 * DAYS;
|
||||||
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
|
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
|
||||||
|
const CLEAN_PER_LOOP = 250;
|
||||||
|
|
||||||
async function cleanup() {
|
async function cleanup() {
|
||||||
|
const repo = getRepository(SavedMessage);
|
||||||
|
|
||||||
const deletedAtThreshold = moment()
|
const deletedAtThreshold = moment()
|
||||||
.subtract(CLEANUP_INTERVAL, "ms")
|
.subtract(CLEANUP_INTERVAL, "ms")
|
||||||
.format(DBDateFormat);
|
.format(DBDateFormat);
|
||||||
|
@ -29,28 +31,39 @@ async function cleanup() {
|
||||||
.subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms")
|
.subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms")
|
||||||
.format(DBDateFormat);
|
.format(DBDateFormat);
|
||||||
|
|
||||||
const query = `
|
// SELECT + DELETE messages in batches
|
||||||
DELETE FROM messages
|
// This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
|
||||||
WHERE (
|
// when a message was being inserted at the same time
|
||||||
deleted_at IS NOT NULL
|
let rows;
|
||||||
AND deleted_at <= ?
|
do {
|
||||||
)
|
rows = await connection.query(
|
||||||
OR (
|
`
|
||||||
posted_at <= ?
|
SELECT id
|
||||||
AND is_permanent = 0
|
FROM messages
|
||||||
)
|
WHERE (
|
||||||
OR (
|
deleted_at IS NOT NULL
|
||||||
is_bot = 1
|
AND deleted_at <= ?
|
||||||
AND posted_at <= ?
|
)
|
||||||
AND is_permanent = 0
|
OR (
|
||||||
)
|
posted_at <= ?
|
||||||
LIMIT ${25_000}
|
AND is_permanent = 0
|
||||||
`;
|
)
|
||||||
|
OR (
|
||||||
|
is_bot = 1
|
||||||
|
AND posted_at <= ?
|
||||||
|
AND is_permanent = 0
|
||||||
|
)
|
||||||
|
LIMIT ${CLEAN_PER_LOOP}
|
||||||
|
`,
|
||||||
|
[deletedAtThreshold, postedAtThreshold, botPostedAtThreshold],
|
||||||
|
);
|
||||||
|
|
||||||
cleanupPromise = (async () => {
|
if (rows.length > 0) {
|
||||||
await connection.query(query, [deletedAtThreshold, postedAtThreshold, botPostedAtThreshold]);
|
await repo.delete({
|
||||||
})();
|
id: In(rows.map(r => r.id)),
|
||||||
await cleanupPromise;
|
});
|
||||||
|
}
|
||||||
|
} while (rows.length === CLEAN_PER_LOOP);
|
||||||
|
|
||||||
setTimeout(cleanup, CLEANUP_INTERVAL);
|
setTimeout(cleanup, CLEANUP_INTERVAL);
|
||||||
}
|
}
|
||||||
|
@ -139,8 +152,8 @@ export class GuildSavedMessages extends BaseGuildRepository {
|
||||||
let query = this.messages
|
let query = this.messages
|
||||||
.createQueryBuilder()
|
.createQueryBuilder()
|
||||||
.where("guild_id = :guild_id", { guild_id: this.guildId })
|
.where("guild_id = :guild_id", { guild_id: this.guildId })
|
||||||
.andWhere("user_id = :user_id", { user_id: userId })
|
|
||||||
.andWhere("channel_id = :channel_id", { channel_id: channelId })
|
.andWhere("channel_id = :channel_id", { channel_id: channelId })
|
||||||
|
.andWhere("user_id = :user_id", { user_id: userId })
|
||||||
.andWhere("id > :afterId", { afterId })
|
.andWhere("id > :afterId", { afterId })
|
||||||
.andWhere("deleted_at IS NULL");
|
.andWhere("deleted_at IS NULL");
|
||||||
|
|
||||||
|
@ -166,8 +179,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
|
||||||
this.toBePermanent.delete(data.id);
|
this.toBePermanent.delete(data.id);
|
||||||
}
|
}
|
||||||
|
|
||||||
await cleanupPromise;
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await this.messages.insert(data);
|
await this.messages.insert(data);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
@ -184,8 +195,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
|
||||||
const existingSavedMsg = await this.find(msg.id);
|
const existingSavedMsg = await this.find(msg.id);
|
||||||
if (existingSavedMsg) return;
|
if (existingSavedMsg) return;
|
||||||
|
|
||||||
await cleanupPromise;
|
|
||||||
|
|
||||||
const savedMessageData = this.msgToSavedMessageData(msg);
|
const savedMessageData = this.msgToSavedMessageData(msg);
|
||||||
const postedAt = moment.utc(msg.timestamp, "x").format("YYYY-MM-DD HH:mm:ss.SSS");
|
const postedAt = moment.utc(msg.timestamp, "x").format("YYYY-MM-DD HH:mm:ss.SSS");
|
||||||
|
|
||||||
|
@ -203,8 +212,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
|
||||||
}
|
}
|
||||||
|
|
||||||
async markAsDeleted(id) {
|
async markAsDeleted(id) {
|
||||||
await cleanupPromise;
|
|
||||||
|
|
||||||
await this.messages
|
await this.messages
|
||||||
.createQueryBuilder("messages")
|
.createQueryBuilder("messages")
|
||||||
.update()
|
.update()
|
||||||
|
@ -228,8 +235,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
|
||||||
* If any messages were marked as deleted, also emits the deleteBulk event.
|
* If any messages were marked as deleted, also emits the deleteBulk event.
|
||||||
*/
|
*/
|
||||||
async markBulkAsDeleted(ids) {
|
async markBulkAsDeleted(ids) {
|
||||||
await cleanupPromise;
|
|
||||||
|
|
||||||
const deletedAt = moment().format("YYYY-MM-DD HH:mm:ss.SSS");
|
const deletedAt = moment().format("YYYY-MM-DD HH:mm:ss.SSS");
|
||||||
|
|
||||||
await this.messages
|
await this.messages
|
||||||
|
@ -256,8 +261,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
|
||||||
const oldMessage = await this.messages.findOne(id);
|
const oldMessage = await this.messages.findOne(id);
|
||||||
if (!oldMessage) return;
|
if (!oldMessage) return;
|
||||||
|
|
||||||
await cleanupPromise;
|
|
||||||
|
|
||||||
const newMessage = { ...oldMessage, data: newData };
|
const newMessage = { ...oldMessage, data: newData };
|
||||||
|
|
||||||
await this.messages.update(
|
await this.messages.update(
|
||||||
|
@ -279,7 +282,6 @@ export class GuildSavedMessages extends BaseGuildRepository {
|
||||||
async setPermanent(id: string) {
|
async setPermanent(id: string) {
|
||||||
const savedMsg = await this.find(id);
|
const savedMsg = await this.find(id);
|
||||||
if (savedMsg) {
|
if (savedMsg) {
|
||||||
await cleanupPromise;
|
|
||||||
await this.messages.update(
|
await this.messages.update(
|
||||||
{ id },
|
{ id },
|
||||||
{
|
{
|
||||||
|
|
Loading…
Add table
Reference in a new issue