Potential optimization to message deletion loop

This commit is contained in:
Dragory 2021-09-12 22:09:29 +03:00
parent b33fd8d604
commit 57470e5a7f
No known key found for this signature in database
GPG key ID: 5F387BA66DF8AAC1

View file

@ -11,7 +11,7 @@ import { SavedMessage } from "../entities/SavedMessage";
const RETENTION_PERIOD = 1 * DAYS; const RETENTION_PERIOD = 1 * DAYS;
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES; const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES; const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES;
const CLEAN_PER_LOOP = 500; const CLEAN_PER_LOOP = 200;
export async function cleanupMessages(): Promise<number> { export async function cleanupMessages(): Promise<number> {
let cleaned = 0; let cleaned = 0;
@ -25,38 +25,64 @@ export async function cleanupMessages(): Promise<number> {
// SELECT + DELETE messages in batches // SELECT + DELETE messages in batches
// This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below // This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
// when a message was being inserted at the same time // when a message was being inserted at the same time
let rows; let ids: string[];
do { do {
rows = await connection.query( const deletedMessageRows = await connection.query(
` `
SELECT id SELECT id
FROM messages FROM messages
WHERE ( WHERE (
deleted_at IS NOT NULL deleted_at IS NOT NULL
AND deleted_at <= ? AND deleted_at <= ?
) )
OR (
posted_at <= ?
AND is_permanent = 0
)
OR (
is_bot = 1
AND posted_at <= ?
AND is_permanent = 0
)
LIMIT ${CLEAN_PER_LOOP} LIMIT ${CLEAN_PER_LOOP}
`, `,
[deletedAtThreshold, postedAtThreshold, botPostedAtThreshold], [deletedAtThreshold],
); );
if (rows.length > 0) { const oldPostedRows = await connection.query(
`
SELECT id
FROM messages
WHERE (
posted_at <= ?
AND is_permanent = 0
)
LIMIT ${CLEAN_PER_LOOP}
`,
[postedAtThreshold],
);
const oldBotPostedRows = await connection.query(
`
SELECT id
FROM messages
WHERE (
is_bot = 1
AND posted_at <= ?
AND is_permanent = 0
)
LIMIT ${CLEAN_PER_LOOP}
`,
[botPostedAtThreshold],
);
ids = Array.from(
new Set([
...deletedMessageRows.map((r) => r.id),
...oldPostedRows.map((r) => r.id),
...oldBotPostedRows.map((r) => r.id),
]),
);
if (ids.length > 0) {
await messagesRepository.delete({ await messagesRepository.delete({
id: In(rows.map((r) => r.id)), id: In(ids),
}); });
} }
cleaned += rows.length; cleaned += ids.length;
} while (rows.length === CLEAN_PER_LOOP); } while (ids.length > 0);
return cleaned; return cleaned;
} }