Add username/nickname history retention periods

backend/src/data/cleanup/messages.ts

@@ -0,0 +1,68 @@
+import { DAYS, DBDateFormat, MINUTES } from "../../utils";
+import { getRepository, In } from "typeorm";
+import { SavedMessage } from "../entities/SavedMessage";
+import moment from "moment-timezone";
+import { connection } from "../db";
+
+/**
+ * How long message edits, deletions, etc. will include the original message content.
+ * This is very heavy storage-wise, so keeping it as low as possible is ideal.
+ */
+const RETENTION_PERIOD = 1 * DAYS;
+const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
+const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES;
+const CLEAN_PER_LOOP = 250;
+
+export async function cleanupMessages(): Promise<number> {
+  let cleaned = 0;
+
+  const messagesRepository = getRepository(SavedMessage);
+
+  const deletedAtThreshold = moment()
+    .subtract(DELETED_MESSAGE_RETENTION_PERIOD, "ms")
+    .format(DBDateFormat);
+  const postedAtThreshold = moment()
+    .subtract(RETENTION_PERIOD, "ms")
+    .format(DBDateFormat);
+  const botPostedAtThreshold = moment()
+    .subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms")
+    .format(DBDateFormat);
+
+  // SELECT + DELETE messages in batches
+  // This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
+  // when a message was being inserted at the same time
+  let rows;
+  do {
+    rows = await connection.query(
+      `
+      SELECT id
+      FROM messages
+      WHERE (
+          deleted_at IS NOT NULL
+          AND deleted_at <= ?
+        )
+         OR (
+          posted_at <= ?
+          AND is_permanent = 0
+        )
+         OR (
+          is_bot = 1
+          AND posted_at <= ?
+          AND is_permanent = 0
+        )
+      LIMIT ${CLEAN_PER_LOOP}
+    `,
+      [deletedAtThreshold, postedAtThreshold, botPostedAtThreshold],
+    );
+
+    if (rows.length > 0) {
+      await messagesRepository.delete({
+        id: In(rows.map(r => r.id)),
+      });
+    }
+
+    cleaned += rows.length;
+  } while (rows.length === CLEAN_PER_LOOP);
+
+  return cleaned;
+}

backend/src/data/cleanup/nicknames.ts

@@ -0,0 +1,41 @@
+import { getRepository, In } from "typeorm";
+import moment from "moment-timezone";
+import { NicknameHistoryEntry } from "../entities/NicknameHistoryEntry";
+import { DAYS, DBDateFormat } from "../../utils";
+import { connection } from "../db";
+
+export const NICKNAME_RETENTION_PERIOD = 30 * DAYS;
+const CLEAN_PER_LOOP = 500;
+
+export async function cleanupNicknames(): Promise<number> {
+  let cleaned = 0;
+
+  const nicknameHistoryRepository = getRepository(NicknameHistoryEntry);
+  const dateThreshold = moment()
+    .subtract(NICKNAME_RETENTION_PERIOD, "ms")
+    .format(DBDateFormat);
+
+  // Clean old nicknames (NICKNAME_RETENTION_PERIOD)
+  let rows;
+  do {
+    rows = await connection.query(
+      `
+      SELECT id
+      FROM nickname_history
+      WHERE timestamp < ?
+      LIMIT ${CLEAN_PER_LOOP}
+    `,
+      [dateThreshold],
+    );
+
+    if (rows.length > 0) {
+      await nicknameHistoryRepository.delete({
+        id: In(rows.map(r => r.id)),
+      });
+    }
+
+    cleaned += rows.length;
+  } while (rows.length === CLEAN_PER_LOOP);
+
+  return cleaned;
+}

backend/src/data/cleanup/usernames.ts

@@ -0,0 +1,45 @@
+import { getRepository, In } from "typeorm";
+import moment from "moment-timezone";
+import { UsernameHistoryEntry } from "../entities/UsernameHistoryEntry";
+import { DAYS, DBDateFormat } from "../../utils";
+import { connection } from "../db";
+
+export const USERNAME_RETENTION_PERIOD = 30 * DAYS;
+const CLEAN_PER_LOOP = 500;
+
+export async function cleanupUsernames(): Promise<number> {
+  let cleaned = 0;
+
+  const usernameHistoryRepository = getRepository(UsernameHistoryEntry);
+  const dateThreshold = moment()
+    .subtract(USERNAME_RETENTION_PERIOD, "ms")
+    .format(DBDateFormat);
+
+  // Clean old usernames (USERNAME_RETENTION_PERIOD)
+  let rows;
+  do {
+    rows = await connection.query(
+      `
+      SELECT id
+      FROM username_history
+      WHERE timestamp < ?
+      LIMIT ${CLEAN_PER_LOOP}
+    `,
+      [dateThreshold],
+    );
+
+    if (rows.length > 0) {
+      console.log(
+        "ids",
+        rows.map(r => r.id),
+      );
+      await usernameHistoryRepository.delete({
+        id: In(rows.map(r => r.id)),
+      });
+    }
+
+    cleaned += rows.length;
+  } while (rows.length === CLEAN_PER_LOOP);
+
+  return cleaned;
+}