feat: add member cache; handle all role changes with RoleManagerPlugin; exit gracefully

backend/src/data/DefaultLogMessages.json

@@ -13,8 +13,8 @@
   "MEMBER_SOFTBAN": "{timestamp} 🔨 {userMention(member)} was softbanned by {userMention(mod)}",
   "MEMBER_JOIN": "{timestamp} 📥 {new} {userMention(member)} joined (created {account_age} ago)",
   "MEMBER_LEAVE": "{timestamp} 📤 {userMention(member)} left the server",
-  "MEMBER_ROLE_ADD": "{timestamp} 🔑 {userMention(member)} received roles: **{roles}**",
-  "MEMBER_ROLE_REMOVE": "{timestamp} 🔑 {userMention(member)} lost roles: **{roles}**",
+  "MEMBER_ROLE_ADD": "{timestamp} 🔑 {userMention(mod)} added roles for {userMention(member)}: **{roles}**",
+  "MEMBER_ROLE_REMOVE": "{timestamp} 🔑 {userMention(mod)} removed roles from {userMention(member)}: **{roles}**",
   "MEMBER_ROLE_CHANGES": "{timestamp} 🔑 {userMention(member)} had role changes: received **{addedRoles}**, lost **{removedRoles}**",
   "MEMBER_NICK_CHANGE": "{timestamp} ✏ {userMention(member)}: nickname changed from **{oldNick}** to **{newNick}**",
   "MEMBER_USERNAME_CHANGE": "{timestamp} ✏ {userMention(user)}: username changed from **{oldName}** to **{newName}**",

backend/src/data/GuildMemberCache.ts

@@ -0,0 +1,101 @@
+import moment from "moment-timezone";
+import { getRepository, Repository } from "typeorm";
+import { Blocker } from "../Blocker";
+import { DBDateFormat, MINUTES } from "../utils";
+import { BaseGuildRepository } from "./BaseGuildRepository";
+import { MemberCacheItem } from "./entities/MemberCacheItem";
+
+const SAVE_PENDING_BLOCKER_KEY = "save-pending" as const;
+
+const DELETION_DELAY = 5 * MINUTES;
+
+type UpdateData = Pick<MemberCacheItem, "username" | "nickname" | "roles">;
+
+export class GuildMemberCache extends BaseGuildRepository {
+  #memberCache: Repository<MemberCacheItem>;
+
+  #pendingUpdates: Map<string, Partial<MemberCacheItem>>;
+
+  #blocker: Blocker;
+
+  constructor(guildId: string) {
+    super(guildId);
+    this.#memberCache = getRepository(MemberCacheItem);
+    this.#pendingUpdates = new Map();
+    this.#blocker = new Blocker();
+  }
+
+  async savePendingUpdates(): Promise<void> {
+    await this.#blocker.waitToBeUnblocked(SAVE_PENDING_BLOCKER_KEY);
+
+    if (this.#pendingUpdates.size === 0) {
+      return;
+    }
+
+    this.#blocker.block(SAVE_PENDING_BLOCKER_KEY);
+
+    const entitiesToSave = Array.from(this.#pendingUpdates.values());
+    this.#pendingUpdates.clear();
+
+    await this.#memberCache.upsert(entitiesToSave, ["guild_id", "user_id"]).finally(() => {
+      this.#blocker.unblock(SAVE_PENDING_BLOCKER_KEY);
+    });
+  }
+
+  async getCachedMemberData(userId: string): Promise<MemberCacheItem | null> {
+    await this.#blocker.waitToBeUnblocked(SAVE_PENDING_BLOCKER_KEY);
+
+    const dbItem = await this.#memberCache.findOne({
+      where: {
+        guild_id: this.guildId,
+        user_id: userId,
+      },
+    });
+    const pendingItem = this.#pendingUpdates.get(userId);
+    if (!dbItem && !pendingItem) {
+      return null;
+    }
+
+    const item = new MemberCacheItem();
+    Object.assign(item, dbItem ?? {});
+    Object.assign(item, pendingItem ?? {});
+    return item;
+  }
+
+  async setCachedMemberData(userId: string, data: UpdateData): Promise<void> {
+    await this.#blocker.waitToBeUnblocked(SAVE_PENDING_BLOCKER_KEY);
+
+    if (!this.#pendingUpdates.has(userId)) {
+      const newItem = new MemberCacheItem();
+      newItem.guild_id = this.guildId;
+      newItem.user_id = userId;
+      this.#pendingUpdates.set(userId, newItem);
+    }
+    Object.assign(this.#pendingUpdates.get(userId)!, data);
+    this.#pendingUpdates.get(userId)!.last_seen = moment().format("YYYY-MM-DD");
+  }
+
+  async markMemberForDeletion(userId: string): Promise<void> {
+    await this.#memberCache.update(
+      {
+        guild_id: this.guildId,
+        user_id: userId,
+      },
+      {
+        delete_at: moment().add(DELETION_DELAY, "ms").format(DBDateFormat),
+      },
+    );
+  }
+
+  async unmarkMemberForDeletion(userId: string): Promise<void> {
+    await this.#memberCache.update(
+      {
+        guild_id: this.guildId,
+        user_id: userId,
+      },
+      {
+        delete_at: null,
+      },
+    );
+  }
+}

backend/src/data/GuildPersistedData.ts

@@ -2,11 +2,6 @@ import { getRepository, Repository } from "typeorm";
 import { BaseGuildRepository } from "./BaseGuildRepository";
 import { PersistedData } from "./entities/PersistedData";
 
-export interface IPartialPersistData {
-  roles?: string[];
-  nickname?: string;
-}
-
 export class GuildPersistedData extends BaseGuildRepository {
   private persistedData: Repository<PersistedData>;
 
@@ -24,11 +19,7 @@ export class GuildPersistedData extends BaseGuildRepository {
     });
   }
 
-  async set(userId: string, data: IPartialPersistData = {}) {
-    const finalData: any = {};
-    if (data.roles) finalData.roles = data.roles.join(",");
-    if (data.nickname) finalData.nickname = data.nickname;
-
+  async set(userId: string, data: Partial<PersistedData> = {}) {
     const existing = await this.find(userId);
     if (existing) {
       await this.persistedData.update(
@@ -36,11 +27,11 @@ export class GuildPersistedData extends BaseGuildRepository {
           guild_id: this.guildId,
           user_id: userId,
         },
-        finalData,
+        data,
       );
     } else {
       await this.persistedData.insert({
-        ...finalData,
+        ...data,
         guild_id: this.guildId,
         user_id: userId,
       });

backend/src/data/MemberCache.ts

@@ -0,0 +1,29 @@
+import moment from "moment-timezone";
+import { getRepository, Repository } from "typeorm";
+import { DAYS } from "../utils";
+import { BaseRepository } from "./BaseRepository";
+import { MemberCacheItem } from "./entities/MemberCacheItem";
+
+const STALE_PERIOD = 90 * DAYS;
+
+export class MemberCache extends BaseRepository {
+  #memberCache: Repository<MemberCacheItem>;
+
+  constructor() {
+    super();
+    this.#memberCache = getRepository(MemberCacheItem);
+  }
+
+  async deleteStaleData(): Promise<void> {
+    const cutoff = moment().subtract(STALE_PERIOD, "ms").format("YYYY-MM-DD");
+    await this.#memberCache.createQueryBuilder().where("last_seen < :cutoff", { cutoff }).delete().execute();
+  }
+
+  async deleteMarkedToBeDeletedEntries(): Promise<void> {
+    await this.#memberCache
+      .createQueryBuilder()
+      .where("delete_at IS NOT NULL AND delete_at <= NOW()")
+      .delete()
+      .execute();
+  }
+}

backend/src/data/entities/MemberCacheItem.ts

@@ -0,0 +1,20 @@
+import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
+
+@Entity("member_cache")
+export class MemberCacheItem {
+  @PrimaryGeneratedColumn() id: number;
+
+  @Column() guild_id: string;
+
+  @Column() user_id: string;
+
+  @Column() username: string;
+
+  @Column({ type: String, nullable: true }) nickname: string | null;
+
+  @Column("simple-json") roles: string[];
+
+  @Column() last_seen: string;
+
+  @Column({ type: String, nullable: true }) delete_at: string | null;
+}

backend/src/data/entities/PersistedData.ts

@@ -14,5 +14,5 @@ export class PersistedData {
 
   @Column() nickname: string;
 
-  @Column() is_voice_muted: number;
+  @Column({ type: "boolean" }) is_voice_muted: boolean;
 }

backend/src/data/loops/expiredMemberCacheDeletionLoop.ts

@@ -0,0 +1,13 @@
+// tslint:disable:no-console
+
+import { HOURS, lazyMemoize } from "../../utils";
+import { MemberCache } from "../MemberCache";
+
+const LOOP_INTERVAL = 6 * HOURS;
+const getMemberCacheRepository = lazyMemoize(() => new MemberCache());
+
+export async function runExpiredMemberCacheDeletionLoop() {
+  console.log("[EXPIRED MEMBER CACHE DELETION LOOP] Deleting stale member cache entries");
+  await getMemberCacheRepository().deleteStaleData();
+  setTimeout(() => runExpiredMemberCacheDeletionLoop(), LOOP_INTERVAL);
+}

backend/src/data/loops/memberCacheDeletionLoop.ts

@@ -0,0 +1,13 @@
+// tslint:disable:no-console
+
+import { lazyMemoize, MINUTES } from "../../utils";
+import { MemberCache } from "../MemberCache";
+
+const LOOP_INTERVAL = 5 * MINUTES;
+const getMemberCacheRepository = lazyMemoize(() => new MemberCache());
+
+export async function runMemberCacheDeletionLoop() {
+  console.log("[MEMBER CACHE DELETION LOOP] Deleting entries marked to be deleted");
+  await getMemberCacheRepository().deleteMarkedToBeDeletedEntries();
+  setTimeout(() => runMemberCacheDeletionLoop(), LOOP_INTERVAL);
+}