perf: move encryption/decryption to a separate thread

backend/src/data/BaseGuildRepository.ts

@@ -1,6 +1,6 @@
 import { BaseRepository } from "./BaseRepository";
 
-export class BaseGuildRepository extends BaseRepository {
+export class BaseGuildRepository<TEntity extends unknown = unknown> extends BaseRepository<TEntity> {
   private static guildInstances: Map<string, any>;
 
   protected guildId: string;

backend/src/data/BaseRepository.ts

@@ -1,4 +1,6 @@
-export class BaseRepository {
+import { asyncMap } from "../utils/async";
+
+export class BaseRepository<TEntity extends unknown = unknown> {
   private nextRelations: string[];
 
   constructor() {
@@ -27,4 +29,26 @@ export class BaseRepository {
     this.nextRelations = [];
     return relations;
   }
+
+  protected async _processEntityFromDB(entity) {
+    // No-op, override in repository
+    return entity;
+  }
+
+  protected async _processEntityToDB(entity) {
+    // No-op, override in repository
+    return entity;
+  }
+
+  protected async processEntityFromDB<T extends TEntity | undefined>(entity: T): Promise<T> {
+    return this._processEntityFromDB(entity);
+  }
+
+  protected async processMultipleEntitiesFromDB<TArr extends TEntity[]>(entities: TArr): Promise<TArr> {
+    return asyncMap(entities, (entity) => this.processEntityFromDB(entity)) as Promise<TArr>;
+  }
+
+  protected async processEntityToDB<T extends Partial<TEntity>>(entity: T): Promise<T> {
+    return this._processEntityToDB(entity);
+  }
 }

backend/src/data/GuildArchives.ts

@@ -6,12 +6,13 @@ import { renderTemplate, TemplateSafeValueContainer } from "../templateFormatter
 import { trimLines } from "../utils";
 import { BaseGuildRepository } from "./BaseGuildRepository";
 import { ArchiveEntry } from "./entities/ArchiveEntry";
-import { SavedMessage } from "./entities/SavedMessage";
 import {
   channelToTemplateSafeChannel,
   guildToTemplateSafeGuild,
   userToTemplateSafeUser,
 } from "../utils/templateSafeObjects";
+import { decryptJson, encryptJson } from "../utils/cryptHelpers";
+import { SavedMessage } from "./entities/SavedMessage";
 
 const DEFAULT_EXPIRY_DAYS = 30;
 
@@ -21,7 +22,7 @@ const MESSAGE_ARCHIVE_HEADER_FORMAT = trimLines(`
 const MESSAGE_ARCHIVE_MESSAGE_FORMAT =
   "[#{channel.name}] [{user.id}] [{timestamp}] {user.username}#{user.discriminator}: {content}{attachments}{stickers}";
 
-export class GuildArchives extends BaseGuildRepository {
+export class GuildArchives extends BaseGuildRepository<ArchiveEntry> {
   protected archives: Repository<ArchiveEntry>;
 
   constructor(guildId) {
@@ -29,11 +30,28 @@ export class GuildArchives extends BaseGuildRepository {
     this.archives = getRepository(ArchiveEntry);
   }
 
+  protected async _processEntityFromDB(entity: ArchiveEntry | undefined) {
+    if (entity == null) {
+      return entity;
+    }
+
+    entity.body = await decryptJson(entity.body as unknown as string);
+    return entity;
+  }
+
+  protected async _processEntityToDB(entity: Partial<ArchiveEntry>) {
+    if (entity.body) {
+      entity.body = (await encryptJson(entity.body)) as any;
+    }
+    return entity;
+  }
+
   async find(id: string): Promise<ArchiveEntry | undefined> {
-    return this.archives.findOne({
+    const result = await this.archives.findOne({
       where: { id },
       relations: this.getRelations(),
     });
+    return this.processEntityFromDB(result);
   }
 
   async makePermanent(id: string): Promise<void> {
@@ -46,23 +64,24 @@ export class GuildArchives extends BaseGuildRepository {
   }
 
   /**
-   * @returns ID of the created entry
+   * @return - ID of the created archive
    */
   async create(body: string, expiresAt?: moment.Moment): Promise<string> {
     if (!expiresAt) {
       expiresAt = moment.utc().add(DEFAULT_EXPIRY_DAYS, "days");
     }
 
-    const result = await this.archives.insert({
+    const data = await this.processEntityToDB({
       guild_id: this.guildId,
       body,
       expires_at: expiresAt.format("YYYY-MM-DD HH:mm:ss"),
     });
+    const result = await this.archives.insert(data);
 
     return result.identifiers[0].id;
   }
 
-  protected async renderLinesFromSavedMessages(savedMessages: SavedMessage[], guild: Guild) {
+  protected async renderLinesFromSavedMessages(savedMessages: SavedMessage[], guild: Guild): Promise<string[]> {
     const msgLines: string[] = [];
     for (const msg of savedMessages) {
       const channel = guild.channels.cache.get(msg.channel_id as Snowflake);
@@ -90,7 +109,14 @@ export class GuildArchives extends BaseGuildRepository {
     return msgLines;
   }
 
-  async createFromSavedMessages(savedMessages: SavedMessage[], guild: Guild, expiresAt?: moment.Moment) {
+  /**
+   * @return - ID of the created archive
+   */
+  async createFromSavedMessages(
+    savedMessages: SavedMessage[],
+    guild: Guild,
+    expiresAt?: moment.Moment,
+  ): Promise<string> {
     if (expiresAt == null) {
       expiresAt = moment.utc().add(DEFAULT_EXPIRY_DAYS, "days");
     }
@@ -111,12 +137,13 @@ export class GuildArchives extends BaseGuildRepository {
     const msgLines = await this.renderLinesFromSavedMessages(savedMessages, guild);
     const messagesStr = msgLines.join("\n");
 
-    const archive = await this.find(archiveId);
+    let archive = await this.find(archiveId);
     if (archive == null) {
       throw new Error("Archive not found");
     }
 
     archive.body += "\n" + messagesStr;
+    archive = await this.processEntityToDB(archive);
 
     await this.archives.update({ id: archiveId }, { body: archive.body });
   }

backend/src/data/GuildSavedMessages.ts

@@ -7,8 +7,11 @@ import { BaseGuildRepository } from "./BaseGuildRepository";
 import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage";
 import { buildEntity } from "./buildEntity";
 import { noop } from "../utils";
+import { decrypt } from "../utils/crypt";
+import { decryptJson, encryptJson } from "../utils/cryptHelpers";
+import { asyncMap } from "../utils/async";
 
-export class GuildSavedMessages extends BaseGuildRepository {
+export class GuildSavedMessages extends BaseGuildRepository<SavedMessage> {
   private messages: Repository<SavedMessage>;
   protected toBePermanent: Set<string>;
 
@@ -22,7 +25,7 @@ export class GuildSavedMessages extends BaseGuildRepository {
     this.toBePermanent = new Set();
   }
 
-  public msgToSavedMessageData(msg: Message): ISavedMessageData {
+  protected msgToSavedMessageData(msg: Message): ISavedMessageData {
     const data: ISavedMessageData = {
       author: {
         username: msg.author.username,
@@ -120,52 +123,38 @@ export class GuildSavedMessages extends BaseGuildRepository {
     return data;
   }
 
-  find(id) {
-    return this.messages
-      .createQueryBuilder()
-      .where("guild_id = :guild_id", { guild_id: this.guildId })
-      .andWhere("id = :id", { id })
-      .andWhere("deleted_at IS NULL")
-      .getOne();
+  protected async _processEntityFromDB(entity: SavedMessage | undefined) {
+    if (entity == null) {
+      return entity;
+    }
+
+    entity.data = await decryptJson(entity.data as unknown as string);
+    return entity;
   }
 
-  getLatestBotMessagesByChannel(channelId, limit) {
-    return this.messages
-      .createQueryBuilder()
-      .where("guild_id = :guild_id", { guild_id: this.guildId })
-      .andWhere("channel_id = :channel_id", { channel_id: channelId })
-      .andWhere("is_bot = 1")
-      .andWhere("deleted_at IS NULL")
-      .orderBy("id", "DESC")
-      .limit(limit)
-      .getMany();
+  protected async _processEntityToDB(entity: Partial<SavedMessage>) {
+    if (entity.data) {
+      entity.data = (await encryptJson(entity.data)) as any;
+    }
+    return entity;
   }
 
-  getLatestByChannelBeforeId(channelId, beforeId, limit) {
-    return this.messages
+  async find(id: string, includeDeleted = false): Promise<SavedMessage | undefined> {
+    let query = this.messages
       .createQueryBuilder()
       .where("guild_id = :guild_id", { guild_id: this.guildId })
-      .andWhere("channel_id = :channel_id", { channel_id: channelId })
-      .andWhere("id < :beforeId", { beforeId })
-      .andWhere("deleted_at IS NULL")
-      .orderBy("id", "DESC")
-      .limit(limit)
-      .getMany();
+      .andWhere("id = :id", { id });
+
+    if (!includeDeleted) {
+      query = query.andWhere("deleted_at IS NULL");
+    }
+
+    const result = await query.getOne();
+
+    return this.processEntityFromDB(result);
   }
 
-  getLatestByChannelAndUser(channelId, userId, limit) {
-    return this.messages
-      .createQueryBuilder()
-      .where("guild_id = :guild_id", { guild_id: this.guildId })
-      .andWhere("channel_id = :channel_id", { channel_id: channelId })
-      .andWhere("user_id = :user_id", { user_id: userId })
-      .andWhere("deleted_at IS NULL")
-      .orderBy("id", "DESC")
-      .limit(limit)
-      .getMany();
-  }
-
-  getUserMessagesByChannelAfterId(userId, channelId, afterId, limit?: number) {
+  async getUserMessagesByChannelAfterId(userId, channelId, afterId, limit?: number): Promise<SavedMessage[]> {
     let query = this.messages
       .createQueryBuilder()
       .where("guild_id = :guild_id", { guild_id: this.guildId })
@@ -178,15 +167,19 @@ export class GuildSavedMessages extends BaseGuildRepository {
       query = query.limit(limit);
     }
 
-    return query.getMany();
+    const results = await query.getMany();
+
+    return this.processMultipleEntitiesFromDB(results);
   }
 
-  getMultiple(messageIds: string[]): Promise<SavedMessage[]> {
-    return this.messages
+  async getMultiple(messageIds: string[]): Promise<SavedMessage[]> {
+    const results = await this.messages
       .createQueryBuilder()
       .where("guild_id = :guild_id", { guild_id: this.guildId })
       .andWhere("id IN (:messageIds)", { messageIds })
       .getMany();
+
+    return this.processMultipleEntitiesFromDB(results);
   }
 
   async createFromMsg(msg: Message, overrides = {}): Promise<void> {
@@ -199,15 +192,18 @@ export class GuildSavedMessages extends BaseGuildRepository {
   }
 
   async createFromMessages(messages: Message[], overrides = {}): Promise<void> {
-    const items = messages.map((msg) => ({ ...this.msgToInsertReadyEntity(msg), ...overrides }));
+    const items = await asyncMap(messages, async (msg) => ({
+      ...(await this.msgToInsertReadyEntity(msg)),
+      ...overrides,
+    }));
     await this.insertBulk(items);
   }
 
-  protected msgToInsertReadyEntity(msg: Message): Partial<SavedMessage> {
+  protected async msgToInsertReadyEntity(msg: Message): Promise<Partial<SavedMessage>> {
     const savedMessageData = this.msgToSavedMessageData(msg);
     const postedAt = moment.utc(msg.createdTimestamp, "x").format("YYYY-MM-DD HH:mm:ss");
 
-    return {
+    return this.processEntityToDB({
       id: msg.id,
       guild_id: (msg.channel as GuildChannel).guild.id,
       channel_id: msg.channel.id,
@@ -215,7 +211,7 @@ export class GuildSavedMessages extends BaseGuildRepository {
       is_bot: msg.author.bot,
       data: savedMessageData,
       posted_at: postedAt,
-    };
+    });
   }
 
   protected async insertBulk(items: Array<Partial<SavedMessage>>): Promise<void> {
@@ -247,7 +243,7 @@ export class GuildSavedMessages extends BaseGuildRepository {
       .andWhere("id = :id", { id })
       .execute();
 
-    const deleted = await this.messages.findOne(id);
+    const deleted = await this.find(id, true);
 
     if (deleted) {
       this.events.emit("delete", [deleted]);
@@ -271,42 +267,40 @@ export class GuildSavedMessages extends BaseGuildRepository {
       .andWhere("deleted_at IS NULL")
       .execute();
 
-    const deleted = await this.messages
+    let deleted = await this.messages
       .createQueryBuilder()
       .where("id IN (:ids)", { ids })
       .andWhere("deleted_at = :deletedAt", { deletedAt })
       .getMany();
+    deleted = await this.processMultipleEntitiesFromDB(deleted);
 
     if (deleted.length) {
       this.events.emit("deleteBulk", [deleted]);
     }
   }
 
-  async saveEdit(id, newData: ISavedMessageData) {
-    const oldMessage = await this.messages.findOne(id);
+  async saveEdit(id, newData: ISavedMessageData): Promise<void> {
+    const oldMessage = await this.find(id);
     if (!oldMessage) return;
 
     const newMessage = { ...oldMessage, data: newData };
 
     // @ts-ignore
-    await this.messages.update(
-      // FIXME?
-      { id },
-      {
-        data: newData as QueryDeepPartialEntity<ISavedMessageData>,
-      },
-    );
+    const updateData = await this.processEntityToDB({
+      data: newData,
+    });
+    await this.messages.update({ id }, updateData);
 
     this.events.emit("update", [newMessage, oldMessage]);
     this.events.emit(`update:${id}`, [newMessage, oldMessage]);
   }
 
-  async saveEditFromMsg(msg: Message) {
+  async saveEditFromMsg(msg: Message): Promise<void> {
     const newData = this.msgToSavedMessageData(msg);
-    return this.saveEdit(msg.id, newData);
+    await this.saveEdit(msg.id, newData);
   }
 
-  async setPermanent(id: string) {
+  async setPermanent(id: string): Promise<void> {
     const savedMsg = await this.find(id);
     if (savedMsg) {
       await this.messages.update(
@@ -320,7 +314,11 @@ export class GuildSavedMessages extends BaseGuildRepository {
     }
   }
 
-  async onceMessageAvailable(id: string, handler: (msg?: SavedMessage) => any, timeout: number = 60 * 1000) {
+  async onceMessageAvailable(
+    id: string,
+    handler: (msg?: SavedMessage) => any,
+    timeout: number = 60 * 1000,
+  ): Promise<void> {
     let called = false;
     let onceEventListener;
     let timeoutFn;

backend/src/data/encryptedJsonTransformer.ts

@@ -1,22 +0,0 @@
-import { ValueTransformer } from "typeorm";
-import { decrypt, encrypt } from "../utils/crypt";
-
-interface EncryptedJsonTransformer<T> extends ValueTransformer {
-  from(dbValue: any): T;
-  to(entityValue: T): any;
-}
-
-export function createEncryptedJsonTransformer<T>(): EncryptedJsonTransformer<T> {
-  return {
-    // Database -> Entity
-    from(dbValue) {
-      const decrypted = decrypt(dbValue);
-      return JSON.parse(decrypted) as T;
-    },
-
-    // Entity -> Database
-    to(entityValue) {
-      return encrypt(JSON.stringify(entityValue));
-    },
-  };
-}

backend/src/data/encryptedTextTransformer.ts

@@ -1,21 +0,0 @@
-import { ValueTransformer } from "typeorm";
-import { decrypt, encrypt } from "../utils/crypt";
-
-interface EncryptedTextTransformer extends ValueTransformer {
-  from(dbValue: any): string;
-  to(entityValue: string): any;
-}
-
-export function createEncryptedTextTransformer(): EncryptedTextTransformer {
-  return {
-    // Database -> Entity
-    from(dbValue) {
-      return decrypt(dbValue);
-    },
-
-    // Entity -> Database
-    to(entityValue) {
-      return encrypt(entityValue);
-    },
-  };
-}

backend/src/data/entities/ArchiveEntry.ts

@@ -1,5 +1,4 @@
 import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
-import { createEncryptedTextTransformer } from "../encryptedTextTransformer";
 
 @Entity("archives")
 export class ArchiveEntry {
@@ -11,7 +10,6 @@ export class ArchiveEntry {
 
   @Column({
     type: "mediumtext",
-    transformer: createEncryptedTextTransformer(),
   })
   body: string;
 

backend/src/data/entities/SavedMessage.ts

@@ -1,6 +1,5 @@
 import { Snowflake } from "discord.js";
 import { Column, Entity, PrimaryColumn } from "typeorm";
-import { createEncryptedJsonTransformer } from "../encryptedJsonTransformer";
 
 export interface ISavedMessageAttachmentData {
   id: Snowflake;
@@ -93,7 +92,6 @@ export class SavedMessage {
 
   @Column({
     type: "mediumtext",
-    transformer: createEncryptedJsonTransformer<ISavedMessageData>(),
   })
   data: ISavedMessageData;