StyAndCo/zeppelin
3
0
Fork 0
mirror of https://github.com/ZeppelinBot/Zeppelin.git synced 2025-03-15 05:41:51 +00:00
Code Releases Activity

perf: move encryption/decryption to a separate thread

Browse source
This commit is contained in:
Dragory 2021-10-09 14:21:23 +03:00
parent 0b337a13a4
commit b7c7e002eb
No known key found for this signature in database
GPG key ID: 5F387BA66DF8AAC1
16 changed files with 310 additions and 147 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

125
backend/package-lock.json generated
View file

@ -46,6 +46,7 @@
"safe-regex": "^2.0.2",
"seedrandom": "^3.0.1",
"strip-combining-marks": "^1.0.0",
"threads": "^1.7.0",
"tlds": "^1.221.1",
"tmp": "0.0.33",
"tsconfig-paths": "^3.9.0",
@ -2234,6 +2235,15 @@
"node": ">=0.8.0"
}
},
"node_modules/esm": {
"version": "3.2.25",
"resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz",
"integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==",
"optional": true,
"engines": {
"node": ">=6"
}
},
"node_modules/esprima": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
@ -2923,6 +2933,17 @@
"node": ">=8"
}
},
"node_modules/is-observable": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-observable/-/is-observable-2.1.0.tgz",
"integrity": "sha512-DailKdLb0WU+xX8K5w7VsJhapwHLZ9jjmazqCJq4X12CTgqq73TKnbRcnSLuXYPOoLQgV5IrD7ePiX/h1vnkBw==",
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/is-path-cwd": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz",
@ -3544,6 +3565,11 @@
"node": ">=0.10.0"
}
},
"node_modules/observable-fns": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/observable-fns/-/observable-fns-0.6.1.tgz",
"integrity": "sha512-9gRK4+sRWzeN6AOewNBTLXir7Zl/i3GB6Yl26gK4flxz8BXVpD3kt8amREmWNb0mxYOGDotvE5a4N+PtGGKdkg=="
},
"node_modules/on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -4983,6 +5009,44 @@
"node": ">=0.8"
}
},
"node_modules/threads": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/threads/-/threads-1.7.0.tgz",
"integrity": "sha512-Mx5NBSHX3sQYR6iI9VYbgHKBLisyB+xROCBGjjWm1O9wb9vfLxdaGtmT/KCjUqMsSNW6nERzCW3T6H43LqjDZQ==",
"dependencies": {
"callsites": "^3.1.0",
"debug": "^4.2.0",
"is-observable": "^2.1.0",
"observable-fns": "^0.6.1"
},
"funding": {
"url": "https://github.com/andywer/threads.js?sponsor=1"
},
"optionalDependencies": {
"tiny-worker": ">= 2"
}
},
"node_modules/threads/node_modules/debug": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
"integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
"dependencies": {
"ms": "2.1.2"
},
"engines": {
"node": ">=6.0"
},
"peerDependenciesMeta": {
"supports-color": {
"optional": true
}
}
},
"node_modules/threads/node_modules/ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
@ -4998,6 +5062,15 @@
"node": ">=4"
}
},
"node_modules/tiny-worker": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/tiny-worker/-/tiny-worker-2.3.0.tgz",
"integrity": "sha512-pJ70wq5EAqTAEl9IkGzA+fN0836rycEuz2Cn6yeZ6FRzlVS5IDOkFHpIoEsksPRQV34GDqXm65+OlnZqUSyK2g==",
"optional": true,
"dependencies": {
"esm": "^3.2.25"
}
},
"node_modules/tlds": {
"version": "1.221.1",
"resolved": "https://registry.npmjs.org/tlds/-/tlds-1.221.1.tgz",
@ -7677,6 +7750,12 @@
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
},
"esm": {
"version": "3.2.25",
"resolved": "https://registry.npmjs.org/esm/-/esm-3.2.25.tgz",
"integrity": "sha512-U1suiZ2oDVWv4zPO56S0NcR5QriEahGtdN2OR6FiOG4WJvcjBVFB0qI4+eKoWFH483PKGuLuu6V8Z4T5g63UVA==",
"optional": true
},
"esprima": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
@ -8195,6 +8274,11 @@
"resolved": "https://registry.npmjs.org/is-obj/-/is-obj-2.0.0.tgz",
"integrity": "sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w=="
},
"is-observable": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/is-observable/-/is-observable-2.1.0.tgz",
"integrity": "sha512-DailKdLb0WU+xX8K5w7VsJhapwHLZ9jjmazqCJq4X12CTgqq73TKnbRcnSLuXYPOoLQgV5IrD7ePiX/h1vnkBw=="
},
"is-path-cwd": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/is-path-cwd/-/is-path-cwd-2.2.0.tgz",
@ -8691,6 +8775,11 @@
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"observable-fns": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/observable-fns/-/observable-fns-0.6.1.tgz",
"integrity": "sha512-9gRK4+sRWzeN6AOewNBTLXir7Zl/i3GB6Yl26gK4flxz8BXVpD3kt8amREmWNb0mxYOGDotvE5a4N+PtGGKdkg=="
},
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
@ -9807,6 +9896,33 @@
"thenify": ">= 3.1.0 < 4"
}
},
"threads": {
"version": "1.7.0",
"resolved": "https://registry.npmjs.org/threads/-/threads-1.7.0.tgz",
"integrity": "sha512-Mx5NBSHX3sQYR6iI9VYbgHKBLisyB+xROCBGjjWm1O9wb9vfLxdaGtmT/KCjUqMsSNW6nERzCW3T6H43LqjDZQ==",
"requires": {
"callsites": "^3.1.0",
"debug": "^4.2.0",
"is-observable": "^2.1.0",
"observable-fns": "^0.6.1",
"tiny-worker": ">= 2"
},
"dependencies": {
"debug": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz",
"integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==",
"requires": {
"ms": "2.1.2"
}
},
"ms": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
}
}
},
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
@ -9819,6 +9935,15 @@
"integrity": "sha1-mcW/VZWJZq9tBtg73zgA3IL67F0=",
"dev": true
},
"tiny-worker": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/tiny-worker/-/tiny-worker-2.3.0.tgz",
"integrity": "sha512-pJ70wq5EAqTAEl9IkGzA+fN0836rycEuz2Cn6yeZ6FRzlVS5IDOkFHpIoEsksPRQV34GDqXm65+OlnZqUSyK2g==",
"optional": true,
"requires": {
"esm": "^3.2.25"
}
},
"tlds": {
"version": "1.221.1",
"resolved": "https://registry.npmjs.org/tlds/-/tlds-1.221.1.tgz",

1
backend/package.json
View file

@ -61,6 +61,7 @@
"safe-regex": "^2.0.2",
"seedrandom": "^3.0.1",
"strip-combining-marks": "^1.0.0",
"threads": "^1.7.0",
"tlds": "^1.221.1",
"tmp": "0.0.33",
"tsconfig-paths": "^3.9.0",

2
backend/src/data/BaseGuildRepository.ts
View file

@ -1,6 +1,6 @@
import { BaseRepository } from "./BaseRepository";
export class BaseGuildRepository extends BaseRepository {
export class BaseGuildRepository<TEntity extends unknown = unknown> extends BaseRepository<TEntity> {
private static guildInstances: Map<string, any>;
protected guildId: string;

26
backend/src/data/BaseRepository.ts
View file

@ -1,4 +1,6 @@
export class BaseRepository {
import { asyncMap } from "../utils/async";
export class BaseRepository<TEntity extends unknown = unknown> {
private nextRelations: string[];
constructor() {
@ -27,4 +29,26 @@ export class BaseRepository {
this.nextRelations = [];
return relations;
}
protected async _processEntityFromDB(entity) {
// No-op, override in repository
return entity;
}
protected async _processEntityToDB(entity) {
// No-op, override in repository
return entity;
}
protected async processEntityFromDB<T extends TEntity | undefined>(entity: T): Promise<T> {
return this._processEntityFromDB(entity);
}
protected async processMultipleEntitiesFromDB<TArr extends TEntity[]>(entities: TArr): Promise<TArr> {
return asyncMap(entities, (entity) => this.processEntityFromDB(entity)) as Promise<TArr>;
}
protected async processEntityToDB<T extends Partial<TEntity>>(entity: T): Promise<T> {
return this._processEntityToDB(entity);
}
}

43
backend/src/data/GuildArchives.ts
View file

@ -6,12 +6,13 @@ import { renderTemplate, TemplateSafeValueContainer } from "../templateFormatter
import { trimLines } from "../utils";
import { BaseGuildRepository } from "./BaseGuildRepository";
import { ArchiveEntry } from "./entities/ArchiveEntry";
import { SavedMessage } from "./entities/SavedMessage";
import {
channelToTemplateSafeChannel,
guildToTemplateSafeGuild,
userToTemplateSafeUser,
} from "../utils/templateSafeObjects";
import { decryptJson, encryptJson } from "../utils/cryptHelpers";
import { SavedMessage } from "./entities/SavedMessage";
const DEFAULT_EXPIRY_DAYS = 30;
@ -21,7 +22,7 @@ const MESSAGE_ARCHIVE_HEADER_FORMAT = trimLines(`
const MESSAGE_ARCHIVE_MESSAGE_FORMAT =
"[#{channel.name}] [{user.id}] [{timestamp}] {user.username}#{user.discriminator}: {content}{attachments}{stickers}";
export class GuildArchives extends BaseGuildRepository {
export class GuildArchives extends BaseGuildRepository<ArchiveEntry> {
protected archives: Repository<ArchiveEntry>;
constructor(guildId) {
@ -29,11 +30,28 @@ export class GuildArchives extends BaseGuildRepository {
this.archives = getRepository(ArchiveEntry);
}
protected async _processEntityFromDB(entity: ArchiveEntry | undefined) {
if (entity == null) {
return entity;
}
entity.body = await decryptJson(entity.body as unknown as string);
return entity;
}
protected async _processEntityToDB(entity: Partial<ArchiveEntry>) {
if (entity.body) {
entity.body = (await encryptJson(entity.body)) as any;
}
return entity;
}
async find(id: string): Promise<ArchiveEntry | undefined> {
return this.archives.findOne({
const result = await this.archives.findOne({
where: { id },
relations: this.getRelations(),
});
return this.processEntityFromDB(result);
}
async makePermanent(id: string): Promise<void> {
@ -46,23 +64,24 @@ export class GuildArchives extends BaseGuildRepository {
}
/**
* @returns ID of the created entry
* @return - ID of the created archive
*/
async create(body: string, expiresAt?: moment.Moment): Promise<string> {
if (!expiresAt) {
expiresAt = moment.utc().add(DEFAULT_EXPIRY_DAYS, "days");
}
const result = await this.archives.insert({
const data = await this.processEntityToDB({
guild_id: this.guildId,
body,
expires_at: expiresAt.format("YYYY-MM-DD HH:mm:ss"),
});
const result = await this.archives.insert(data);
return result.identifiers[0].id;
}
protected async renderLinesFromSavedMessages(savedMessages: SavedMessage[], guild: Guild) {
protected async renderLinesFromSavedMessages(savedMessages: SavedMessage[], guild: Guild): Promise<string[]> {
const msgLines: string[] = [];
for (const msg of savedMessages) {
const channel = guild.channels.cache.get(msg.channel_id as Snowflake);
@ -90,7 +109,14 @@ export class GuildArchives extends BaseGuildRepository {
return msgLines;
}
async createFromSavedMessages(savedMessages: SavedMessage[], guild: Guild, expiresAt?: moment.Moment) {
/**
* @return - ID of the created archive
*/
async createFromSavedMessages(
savedMessages: SavedMessage[],
guild: Guild,
expiresAt?: moment.Moment,
): Promise<string> {
if (expiresAt == null) {
expiresAt = moment.utc().add(DEFAULT_EXPIRY_DAYS, "days");
}
@ -111,12 +137,13 @@ export class GuildArchives extends BaseGuildRepository {
const msgLines = await this.renderLinesFromSavedMessages(savedMessages, guild);
const messagesStr = msgLines.join("\n");
const archive = await this.find(archiveId);
let archive = await this.find(archiveId);
if (archive == null) {
throw new Error("Archive not found");
}
archive.body += "\n" + messagesStr;
archive = await this.processEntityToDB(archive);
await this.archives.update({ id: archiveId }, { body: archive.body });
}

122
backend/src/data/GuildSavedMessages.ts
View file

@ -7,8 +7,11 @@ import { BaseGuildRepository } from "./BaseGuildRepository";
import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage";
import { buildEntity } from "./buildEntity";
import { noop } from "../utils";
import { decrypt } from "../utils/crypt";
import { decryptJson, encryptJson } from "../utils/cryptHelpers";
import { asyncMap } from "../utils/async";
export class GuildSavedMessages extends BaseGuildRepository {
export class GuildSavedMessages extends BaseGuildRepository<SavedMessage> {
private messages: Repository<SavedMessage>;
protected toBePermanent: Set<string>;
@ -22,7 +25,7 @@ export class GuildSavedMessages extends BaseGuildRepository {
this.toBePermanent = new Set();
}
public msgToSavedMessageData(msg: Message): ISavedMessageData {
protected msgToSavedMessageData(msg: Message): ISavedMessageData {
const data: ISavedMessageData = {
author: {
username: msg.author.username,
@ -120,52 +123,38 @@ export class GuildSavedMessages extends BaseGuildRepository {
return data;
}
find(id) {
return this.messages
.createQueryBuilder()
.where("guild_id = :guild_id", { guild_id: this.guildId })
.andWhere("id = :id", { id })
.andWhere("deleted_at IS NULL")
.getOne();
protected async _processEntityFromDB(entity: SavedMessage | undefined) {
if (entity == null) {
return entity;
}
entity.data = await decryptJson(entity.data as unknown as string);
return entity;
}
getLatestBotMessagesByChannel(channelId, limit) {
return this.messages
.createQueryBuilder()
.where("guild_id = :guild_id", { guild_id: this.guildId })
.andWhere("channel_id = :channel_id", { channel_id: channelId })
.andWhere("is_bot = 1")
.andWhere("deleted_at IS NULL")
.orderBy("id", "DESC")
.limit(limit)
.getMany();
protected async _processEntityToDB(entity: Partial<SavedMessage>) {
if (entity.data) {
entity.data = (await encryptJson(entity.data)) as any;
}
return entity;
}
getLatestByChannelBeforeId(channelId, beforeId, limit) {
return this.messages
async find(id: string, includeDeleted = false): Promise<SavedMessage | undefined> {
let query = this.messages
.createQueryBuilder()
.where("guild_id = :guild_id", { guild_id: this.guildId })
.andWhere("channel_id = :channel_id", { channel_id: channelId })
.andWhere("id < :beforeId", { beforeId })
.andWhere("deleted_at IS NULL")
.orderBy("id", "DESC")
.limit(limit)
.getMany();
.andWhere("id = :id", { id });
if (!includeDeleted) {
query = query.andWhere("deleted_at IS NULL");
}
const result = await query.getOne();
return this.processEntityFromDB(result);
}
getLatestByChannelAndUser(channelId, userId, limit) {
return this.messages
.createQueryBuilder()
.where("guild_id = :guild_id", { guild_id: this.guildId })
.andWhere("channel_id = :channel_id", { channel_id: channelId })
.andWhere("user_id = :user_id", { user_id: userId })
.andWhere("deleted_at IS NULL")
.orderBy("id", "DESC")
.limit(limit)
.getMany();
}
getUserMessagesByChannelAfterId(userId, channelId, afterId, limit?: number) {
async getUserMessagesByChannelAfterId(userId, channelId, afterId, limit?: number): Promise<SavedMessage[]> {
let query = this.messages
.createQueryBuilder()
.where("guild_id = :guild_id", { guild_id: this.guildId })
@ -178,15 +167,19 @@ export class GuildSavedMessages extends BaseGuildRepository {
query = query.limit(limit);
}
return query.getMany();
const results = await query.getMany();
return this.processMultipleEntitiesFromDB(results);
}
getMultiple(messageIds: string[]): Promise<SavedMessage[]> {
return this.messages
async getMultiple(messageIds: string[]): Promise<SavedMessage[]> {
const results = await this.messages
.createQueryBuilder()
.where("guild_id = :guild_id", { guild_id: this.guildId })
.andWhere("id IN (:messageIds)", { messageIds })
.getMany();
return this.processMultipleEntitiesFromDB(results);
}
async createFromMsg(msg: Message, overrides = {}): Promise<void> {
@ -199,15 +192,18 @@ export class GuildSavedMessages extends BaseGuildRepository {
}
async createFromMessages(messages: Message[], overrides = {}): Promise<void> {
const items = messages.map((msg) => ({ ...this.msgToInsertReadyEntity(msg), ...overrides }));
const items = await asyncMap(messages, async (msg) => ({
...(await this.msgToInsertReadyEntity(msg)),
...overrides,
}));
await this.insertBulk(items);
}
protected msgToInsertReadyEntity(msg: Message): Partial<SavedMessage> {
protected async msgToInsertReadyEntity(msg: Message): Promise<Partial<SavedMessage>> {
const savedMessageData = this.msgToSavedMessageData(msg);
const postedAt = moment.utc(msg.createdTimestamp, "x").format("YYYY-MM-DD HH:mm:ss");
return {
return this.processEntityToDB({
id: msg.id,
guild_id: (msg.channel as GuildChannel).guild.id,
channel_id: msg.channel.id,
@ -215,7 +211,7 @@ export class GuildSavedMessages extends BaseGuildRepository {
is_bot: msg.author.bot,
data: savedMessageData,
posted_at: postedAt,
};
});
}
protected async insertBulk(items: Array<Partial<SavedMessage>>): Promise<void> {
@ -247,7 +243,7 @@ export class GuildSavedMessages extends BaseGuildRepository {
.andWhere("id = :id", { id })
.execute();
const deleted = await this.messages.findOne(id);
const deleted = await this.find(id, true);
if (deleted) {
this.events.emit("delete", [deleted]);
@ -271,42 +267,40 @@ export class GuildSavedMessages extends BaseGuildRepository {
.andWhere("deleted_at IS NULL")
.execute();
const deleted = await this.messages
let deleted = await this.messages
.createQueryBuilder()
.where("id IN (:ids)", { ids })
.andWhere("deleted_at = :deletedAt", { deletedAt })
.getMany();
deleted = await this.processMultipleEntitiesFromDB(deleted);
if (deleted.length) {
this.events.emit("deleteBulk", [deleted]);
}
}
async saveEdit(id, newData: ISavedMessageData) {
const oldMessage = await this.messages.findOne(id);
async saveEdit(id, newData: ISavedMessageData): Promise<void> {
const oldMessage = await this.find(id);
if (!oldMessage) return;
const newMessage = { ...oldMessage, data: newData };
// @ts-ignore
await this.messages.update(
// FIXME?
{ id },
{
data: newData as QueryDeepPartialEntity<ISavedMessageData>,
},
);
const updateData = await this.processEntityToDB({
data: newData,
});
await this.messages.update({ id }, updateData);
this.events.emit("update", [newMessage, oldMessage]);
this.events.emit(`update:${id}`, [newMessage, oldMessage]);
}
async saveEditFromMsg(msg: Message) {
async saveEditFromMsg(msg: Message): Promise<void> {
const newData = this.msgToSavedMessageData(msg);
return this.saveEdit(msg.id, newData);
await this.saveEdit(msg.id, newData);
}
async setPermanent(id: string) {
async setPermanent(id: string): Promise<void> {
const savedMsg = await this.find(id);
if (savedMsg) {
await this.messages.update(
@ -320,7 +314,11 @@ export class GuildSavedMessages extends BaseGuildRepository {
}
}
async onceMessageAvailable(id: string, handler: (msg?: SavedMessage) => any, timeout: number = 60 * 1000) {
async onceMessageAvailable(
id: string,
handler: (msg?: SavedMessage) => any,
timeout: number = 60 * 1000,
): Promise<void> {
let called = false;
let onceEventListener;
let timeoutFn;

22
backend/src/data/encryptedJsonTransformer.ts
View file

@ -1,22 +0,0 @@
import { ValueTransformer } from "typeorm";
import { decrypt, encrypt } from "../utils/crypt";
interface EncryptedJsonTransformer<T> extends ValueTransformer {
from(dbValue: any): T;
to(entityValue: T): any;
}
export function createEncryptedJsonTransformer<T>(): EncryptedJsonTransformer<T> {
return {
// Database -> Entity
from(dbValue) {
const decrypted = decrypt(dbValue);
return JSON.parse(decrypted) as T;
},
// Entity -> Database
to(entityValue) {
return encrypt(JSON.stringify(entityValue));
},
};
}

21
backend/src/data/encryptedTextTransformer.ts
View file

@ -1,21 +0,0 @@
import { ValueTransformer } from "typeorm";
import { decrypt, encrypt } from "../utils/crypt";
interface EncryptedTextTransformer extends ValueTransformer {
from(dbValue: any): string;
to(entityValue: string): any;
}
export function createEncryptedTextTransformer(): EncryptedTextTransformer {
return {
// Database -> Entity
from(dbValue) {
return decrypt(dbValue);
},
// Entity -> Database
to(entityValue) {
return encrypt(entityValue);
},
};
}

2
backend/src/data/entities/ArchiveEntry.ts
View file

@ -1,5 +1,4 @@
import { Column, Entity, PrimaryGeneratedColumn } from "typeorm";
import { createEncryptedTextTransformer } from "../encryptedTextTransformer";
@Entity("archives")
export class ArchiveEntry {
@ -11,7 +10,6 @@ export class ArchiveEntry {
@Column({
type: "mediumtext",
transformer: createEncryptedTextTransformer(),
})
body: string;

2
backend/src/data/entities/SavedMessage.ts
View file

@ -1,6 +1,5 @@
import { Snowflake } from "discord.js";
import { Column, Entity, PrimaryColumn } from "typeorm";
import { createEncryptedJsonTransformer } from "../encryptedJsonTransformer";
export interface ISavedMessageAttachmentData {
id: Snowflake;
@ -93,7 +92,6 @@ export class SavedMessage {
@Column({
type: "mediumtext",
transformer: createEncryptedJsonTransformer<ISavedMessageData>(),
})
data: ISavedMessageData;

4
backend/src/migrations/1600283341726-EncryptExistingMessages.ts
View file

@ -9,7 +9,7 @@ export class EncryptExistingMessages1600283341726 implements MigrationInterface
// 2. Encrypt all permanent messages
const messages = await queryRunner.query("SELECT id, data FROM messages");
for (const message of messages) {
const encryptedData = encrypt(message.data);
const encryptedData = await encrypt(message.data);
await queryRunner.query("UPDATE messages SET data = ? WHERE id = ?", [encryptedData, message.id]);
}
}
@ -18,7 +18,7 @@ export class EncryptExistingMessages1600283341726 implements MigrationInterface
// Decrypt all messages
const messages = await queryRunner.query("SELECT id, data FROM messages");
for (const message of messages) {
const decryptedData = decrypt(message.data);
const decryptedData = await decrypt(message.data);
await queryRunner.query("UPDATE messages SET data = ? WHERE id = ?", [decryptedData, message.id]);
}
}

4
backend/src/migrations/1600285077890-EncryptArchives.ts
View file

@ -5,7 +5,7 @@ export class EncryptArchives1600285077890 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<any> {
const archives = await queryRunner.query("SELECT id, body FROM archives");
for (const archive of archives) {
const encryptedBody = encrypt(archive.body);
const encryptedBody = await encrypt(archive.body);
await queryRunner.query("UPDATE archives SET body = ? WHERE id = ?", [encryptedBody, archive.id]);
}
}
@ -13,7 +13,7 @@ export class EncryptArchives1600285077890 implements MigrationInterface {
public async down(queryRunner: QueryRunner): Promise<any> {
const archives = await queryRunner.query("SELECT id, body FROM archives");
for (const archive of archives) {
const decryptedBody = decrypt(archive.body);
const decryptedBody = await decrypt(archive.body);
await queryRunner.query("UPDATE archives SET body = ? WHERE id = ?", [decryptedBody, archive.id]);
}
}

6
backend/src/utils/crypt.test.ts
View file

@ -1,9 +1,9 @@
import test from "ava";
import { decrypt, encrypt } from "./crypt";
test("encrypt() followed by decrypt()", (t) => {
test("encrypt() followed by decrypt()", async (t) => {
const original = "banana 123 👀 💕"; // Includes emojis to verify utf8 stuff works
const encrypted = encrypt(original);
const decrypted = decrypt(encrypted);
const encrypted = await encrypt(original);
const decrypted = await decrypt(encrypted);
t.is(decrypted, original);
});

35
backend/src/utils/crypt.ts
View file

@ -1,5 +1,6 @@
import crypto from "crypto";
import { spawn, Worker } from "threads";
import "../loadEnv";
import type { CryptFns } from "./cryptWorker";
if (!process.env.KEY) {
// tslint:disable-next-line:no-console
@ -8,27 +9,19 @@ if (!process.env.KEY) {
}
const KEY = process.env.KEY;
const ALGORITHM = "aes-256-gcm";
let workerPromise: Promise<CryptFns> | null = null;
export function encrypt(str) {
// Based on https://gist.github.com/rjz/15baffeab434b8125ca4d783f4116d81
const iv = crypto.randomBytes(16);
const cipher = crypto.createCipheriv(ALGORITHM, KEY, iv);
let encrypted = cipher.update(str, "utf8", "base64");
encrypted += cipher.final("base64");
return `${iv.toString("base64")}.${cipher.getAuthTag().toString("base64")}.${encrypted}`;
async function getWorker(): Promise<CryptFns> {
if (workerPromise == null) {
workerPromise = spawn(new Worker("./cryptWorker")) as unknown as Promise<CryptFns>;
}
return workerPromise;
}
export function decrypt(encrypted) {
// Based on https://gist.github.com/rjz/15baffeab434b8125ca4d783f4116d81
const [iv, authTag, encryptedStr] = encrypted.split(".");
const decipher = crypto.createDecipheriv(ALGORITHM, KEY, Buffer.from(iv, "base64"));
decipher.setAuthTag(Buffer.from(authTag, "base64"));
let decrypted = decipher.update(encryptedStr, "base64", "utf8");
decrypted += decipher.final("utf8");
return decrypted;
export async function encrypt(data: string) {
return (await getWorker()).encrypt(data, KEY);
}
export async function decrypt(data: string) {
return (await getWorker()).decrypt(data, KEY);
}

11
backend/src/utils/cryptHelpers.ts Normal file
View file

@ -0,0 +1,11 @@
import { decrypt, encrypt } from "./crypt";
export async function encryptJson(obj: any): Promise<string> {
const serialized = JSON.stringify(obj);
return encrypt(serialized);
}
export async function decryptJson<T extends unknown>(encrypted: string): Promise<T> {
const decrypted = await decrypt(encrypted);
return JSON.parse(decrypted);
}

31
backend/src/utils/cryptWorker.ts Normal file
View file

@ -0,0 +1,31 @@
import crypto from "crypto";
import { expose } from "threads";
const ALGORITHM = "aes-256-gcm";
function encrypt(str, key) {
// Based on https://gist.github.com/rjz/15baffeab434b8125ca4d783f4116d81
const iv = crypto.randomBytes(16);
const cipher = crypto.createCipheriv(ALGORITHM, key, iv);
let encrypted = cipher.update(str, "utf8", "base64");
encrypted += cipher.final("base64");
return `${iv.toString("base64")}.${cipher.getAuthTag().toString("base64")}.${encrypted}`;
}
function decrypt(encrypted, key) {
// Based on https://gist.github.com/rjz/15baffeab434b8125ca4d783f4116d81
const [iv, authTag, encryptedStr] = encrypted.split(".");
const decipher = crypto.createDecipheriv(ALGORITHM, key, Buffer.from(iv, "base64"));
decipher.setAuthTag(Buffer.from(authTag, "base64"));
let decrypted = decipher.update(encryptedStr, "base64", "utf8");
decrypted += decipher.final("utf8");
return decrypted;
}
const toExpose = { encrypt, decrypt };
expose(toExpose);
export type CryptFns = typeof toExpose;