Add config retention period / cleanup

backend/src/data/Configs.ts

@@ -1,15 +1,22 @@
 import { Config } from "./entities/Config";
-import {
-  getConnection,
-  getRepository,
-  Repository,
-  Transaction,
-  TransactionManager,
-  TransactionRepository,
-} from "typeorm";
-import { BaseGuildRepository } from "./BaseGuildRepository";
+import { getRepository, Repository } from "typeorm";
 import { connection } from "./db";
 import { BaseRepository } from "./BaseRepository";
+import { isAPI } from "../globals";
+import { HOURS, SECONDS } from "../utils";
+import { cleanupConfigs } from "./cleanup/configs";
+
+if (isAPI()) {
+  const CLEANUP_INTERVAL = 1 * HOURS;
+
+  async function cleanup() {
+    await cleanupConfigs();
+    setTimeout(cleanup, CLEANUP_INTERVAL);
+  }
+
+  // Start first cleanup 30 seconds after startup
+  setTimeout(cleanup, 30 * SECONDS);
+}
 
 export class Configs extends BaseRepository {
   private configs: Repository<Config>;

backend/src/data/cleanup/configs.ts

@@ -0,0 +1,96 @@
+import { connection } from "../db";
+import { getRepository, In } from "typeorm";
+import { Config } from "../entities/Config";
+import moment from "moment-timezone";
+import { DBDateFormat } from "../../utils";
+
+const CLEAN_PER_LOOP = 50;
+
+export async function cleanupConfigs() {
+  const configRepository = getRepository(Config);
+
+  let cleaned = 0;
+  let rows;
+
+  // >1 month old: 1 config retained per month
+  const oneMonthCutoff = moment()
+    .subtract(30, "days")
+    .format(DBDateFormat);
+  do {
+    rows = await connection.query(
+      `
+      WITH _configs
+      AS (
+        SELECT
+          id,
+          \`key\`,
+          YEAR(edited_at) AS \`year\`,
+          MONTH(edited_at) AS \`month\`,
+          ROW_NUMBER() OVER (
+            PARTITION BY \`key\`, \`year\`, \`month\`
+            ORDER BY edited_at
+          ) AS row_num
+        FROM
+          configs
+        WHERE
+          is_active = 0
+          AND edited_at < ?
+      )
+      SELECT *
+      FROM _configs
+      WHERE row_num > 1
+    `,
+      [oneMonthCutoff],
+    );
+
+    if (rows.length > 0) {
+      await configRepository.delete({
+        id: In(rows.map(r => r.id)),
+      });
+    }
+
+    cleaned += rows.length;
+  } while (rows.length === CLEAN_PER_LOOP);
+
+  // >2 weeks old: 1 config retained per day
+  const twoWeekCutoff = moment()
+    .subtract(2, "weeks")
+    .format(DBDateFormat);
+  do {
+    rows = await connection.query(
+      `
+      WITH _configs
+      AS (
+        SELECT
+          id,
+          \`key\`,
+          DATE(edited_at) AS \`date\`,
+          ROW_NUMBER() OVER (
+            PARTITION BY \`key\`, \`date\`
+            ORDER BY edited_at
+          ) AS row_num
+        FROM
+          configs
+        WHERE
+          is_active = 0
+          AND edited_at < ?
+          AND edited_at >= ?
+      )
+      SELECT *
+      FROM _configs
+      WHERE row_num > 1
+    `,
+      [twoWeekCutoff, oneMonthCutoff],
+    );
+
+    if (rows.length > 0) {
+      await configRepository.delete({
+        id: In(rows.map(r => r.id)),
+      });
+    }
+
+    cleaned += rows.length;
+  } while (rows.length === CLEAN_PER_LOOP);
+
+  return cleaned;
+}