Potential optimization to message deletion loop
This commit is contained in:
parent
b33fd8d604
commit
57470e5a7f
1 changed files with 46 additions and 20 deletions
|
@ -11,7 +11,7 @@ import { SavedMessage } from "../entities/SavedMessage";
|
||||||
const RETENTION_PERIOD = 1 * DAYS;
|
const RETENTION_PERIOD = 1 * DAYS;
|
||||||
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
|
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
|
||||||
const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES;
|
const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES;
|
||||||
const CLEAN_PER_LOOP = 500;
|
const CLEAN_PER_LOOP = 200;
|
||||||
|
|
||||||
export async function cleanupMessages(): Promise<number> {
|
export async function cleanupMessages(): Promise<number> {
|
||||||
let cleaned = 0;
|
let cleaned = 0;
|
||||||
|
@ -25,38 +25,64 @@ export async function cleanupMessages(): Promise<number> {
|
||||||
// SELECT + DELETE messages in batches
|
// SELECT + DELETE messages in batches
|
||||||
// This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
|
// This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
|
||||||
// when a message was being inserted at the same time
|
// when a message was being inserted at the same time
|
||||||
let rows;
|
let ids: string[];
|
||||||
do {
|
do {
|
||||||
rows = await connection.query(
|
const deletedMessageRows = await connection.query(
|
||||||
`
|
`
|
||||||
SELECT id
|
SELECT id
|
||||||
FROM messages
|
FROM messages
|
||||||
WHERE (
|
WHERE (
|
||||||
deleted_at IS NOT NULL
|
deleted_at IS NOT NULL
|
||||||
AND deleted_at <= ?
|
AND deleted_at <= ?
|
||||||
)
|
)
|
||||||
OR (
|
|
||||||
posted_at <= ?
|
|
||||||
AND is_permanent = 0
|
|
||||||
)
|
|
||||||
OR (
|
|
||||||
is_bot = 1
|
|
||||||
AND posted_at <= ?
|
|
||||||
AND is_permanent = 0
|
|
||||||
)
|
|
||||||
LIMIT ${CLEAN_PER_LOOP}
|
LIMIT ${CLEAN_PER_LOOP}
|
||||||
`,
|
`,
|
||||||
[deletedAtThreshold, postedAtThreshold, botPostedAtThreshold],
|
[deletedAtThreshold],
|
||||||
);
|
);
|
||||||
|
|
||||||
if (rows.length > 0) {
|
const oldPostedRows = await connection.query(
|
||||||
|
`
|
||||||
|
SELECT id
|
||||||
|
FROM messages
|
||||||
|
WHERE (
|
||||||
|
posted_at <= ?
|
||||||
|
AND is_permanent = 0
|
||||||
|
)
|
||||||
|
LIMIT ${CLEAN_PER_LOOP}
|
||||||
|
`,
|
||||||
|
[postedAtThreshold],
|
||||||
|
);
|
||||||
|
|
||||||
|
const oldBotPostedRows = await connection.query(
|
||||||
|
`
|
||||||
|
SELECT id
|
||||||
|
FROM messages
|
||||||
|
WHERE (
|
||||||
|
is_bot = 1
|
||||||
|
AND posted_at <= ?
|
||||||
|
AND is_permanent = 0
|
||||||
|
)
|
||||||
|
LIMIT ${CLEAN_PER_LOOP}
|
||||||
|
`,
|
||||||
|
[botPostedAtThreshold],
|
||||||
|
);
|
||||||
|
|
||||||
|
ids = Array.from(
|
||||||
|
new Set([
|
||||||
|
...deletedMessageRows.map((r) => r.id),
|
||||||
|
...oldPostedRows.map((r) => r.id),
|
||||||
|
...oldBotPostedRows.map((r) => r.id),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (ids.length > 0) {
|
||||||
await messagesRepository.delete({
|
await messagesRepository.delete({
|
||||||
id: In(rows.map((r) => r.id)),
|
id: In(ids),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
cleaned += rows.length;
|
cleaned += ids.length;
|
||||||
} while (rows.length === CLEAN_PER_LOOP);
|
} while (ids.length > 0);
|
||||||
|
|
||||||
return cleaned;
|
return cleaned;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Add table
Reference in a new issue