Add MessageSaver plugin. Fix some buggy queries.

migrations/1543053430712-CreateMessagesTable.ts

@@ -0,0 +1,72 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreateMessagesTable1543053430712 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.createTable(
+      new Table({
+        name: "messages",
+        columns: [
+          {
+            name: "id",
+            type: "bigint",
+            unsigned: true,
+            isPrimary: true
+          },
+          {
+            name: "guild_id",
+            type: "bigint",
+            unsigned: true
+          },
+          {
+            name: "channel_id",
+            type: "bigint",
+            unsigned: true
+          },
+          {
+            name: "user_id",
+            type: "bigint",
+            unsigned: true
+          },
+          {
+            name: "is_bot",
+            type: "tinyint",
+            unsigned: true
+          },
+          {
+            name: "data",
+            type: "mediumtext"
+          },
+          {
+            name: "posted_at",
+            type: "datetime(3)"
+          },
+          {
+            name: "deleted_at",
+            type: "datetime(3)",
+            isNullable: true,
+            default: null
+          },
+          {
+            name: "is_permanent",
+            type: "tinyint",
+            unsigned: true,
+            default: 0
+          }
+        ],
+        indices: [
+          { columnNames: ["guild_id"] },
+          { columnNames: ["channel_id"] },
+          { columnNames: ["user_id"] },
+          { columnNames: ["is_bot"] },
+          { columnNames: ["posted_at"] },
+          { columnNames: ["deleted_at"] },
+          { columnNames: ["is_permanent"] }
+        ]
+      })
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropTable("messages");
+  }
+}

src/data/GuildArchives.ts

@@ -21,7 +21,8 @@ export class GuildArchives extends BaseRepository {
   private deleteExpiredArchives() {
     this.archives
       .createQueryBuilder()
-      .where("expires_at <= NOW()")
+      .where("guild_id = :guild_id", { guild_id: this.guildId })
+      .andWhere("expires_at <= NOW()")
       .delete()
       .execute();
   }

src/data/GuildMutes.ts

@@ -15,8 +15,8 @@ export class GuildMutes extends BaseRepository {
     return this.mutes
       .createQueryBuilder("mutes")
       .where("guild_id = :guild_id", { guild_id: this.guildId })
-      .where("expires_at IS NOT NULL")
-      .where("expires_at <= NOW()")
+      .andWhere("expires_at IS NOT NULL")
+      .andWhere("expires_at <= NOW()")
       .getMany();
   }
 

src/data/GuildSavedMessages.ts

@@ -0,0 +1,98 @@
+import { Brackets, getRepository, Repository } from "typeorm";
+import { BaseRepository } from "./BaseRepository";
+import { ISavedMessageData, SavedMessage } from "./entities/SavedMessage";
+import EventEmitter from "events";
+
+const CLEANUP_INTERVAL = 5 * 60 * 1000;
+
+const RETENTION_PERIOD = 7 * 24 * 60 * 60 * 1000; // 1 week
+
+export class GuildSavedMessages extends BaseRepository {
+  private messages: Repository<SavedMessage>;
+  public events: EventEmitter;
+
+  constructor(guildId) {
+    super(guildId);
+    this.messages = getRepository(SavedMessage);
+    this.events = new EventEmitter();
+
+    this.cleanup();
+    setInterval(() => this.cleanup(), CLEANUP_INTERVAL);
+  }
+
+  async cleanup() {
+    await this.messages
+      .createQueryBuilder("messages")
+      .where("guild_id = :guild_id", { guild_id: this.guildId })
+      .andWhere(
+        new Brackets(qb => {
+          // Clear deleted messages
+          qb.orWhere(
+            new Brackets(qb2 => {
+              qb2.where("deleted_at IS NOT NULL");
+              qb2.andWhere(`deleted_at <= (NOW() - INTERVAL ${CLEANUP_INTERVAL}000 MICROSECOND)`);
+            })
+          );
+
+          // Clear old messages
+          qb.orWhere(
+            new Brackets(qb2 => {
+              qb2.where("is_permanent = 0");
+              qb2.andWhere(`posted_at <= (NOW() - INTERVAL ${RETENTION_PERIOD}000 MICROSECOND)`);
+            })
+          );
+        })
+      )
+      .delete()
+      .execute();
+  }
+
+  find(id) {
+    return this.messages.findOne({
+      where: {
+        guild_id: this.guildId,
+        id
+      }
+    });
+  }
+
+  async create(data) {
+    try {
+      await this.messages.insert({ ...data, guild_id: this.guildId });
+    } catch (e) {
+      return;
+    }
+
+    const inserted = await this.messages.findOne(data.id);
+    this.events.emit("create", [inserted]);
+  }
+
+  async markAsDeleted(id) {
+    await this.messages
+      .createQueryBuilder("messages")
+      .update()
+      .set({
+        deleted_at: () => "NOW(3)"
+      })
+      .where("guild_id = :guild_id", { guild_id: this.guildId })
+      .andWhere("id = :id", { id })
+      .execute();
+
+    const deleted = await this.messages.findOne(id);
+    this.events.emit("delete", [deleted]);
+  }
+
+  async edit(id, newData: ISavedMessageData) {
+    const oldMessage = await this.messages.findOne(id);
+    const newMessage = { ...oldMessage, data: newData };
+
+    await this.messages.update(
+      { id },
+      {
+        data: newData
+      }
+    );
+
+    this.events.emit("edit", [newMessage, oldMessage]);
+  }
+}

src/data/GuildTags.ts

@@ -31,7 +31,7 @@ export class GuildTags extends BaseRepository {
           created_at: () => "NOW()"
         })
         .where("guild_id = :guildId", { guildId: this.guildId })
-        .where("tag = :tag", { tag })
+        .andWhere("tag = :tag", { tag })
         .execute();
     } else {
       await this.tags.insert({

src/data/entities/SavedMessage.ts

@@ -0,0 +1,33 @@
+import { Entity, Column, PrimaryColumn } from "typeorm";
+import { Message } from "eris";
+
+export interface ISavedMessageData {
+  attachments: object[];
+  author: {
+    username: string;
+    discriminator: string;
+  };
+  content: string;
+  embeds: object[];
+}
+
+@Entity("messages")
+export class SavedMessage {
+  @Column()
+  @PrimaryColumn()
+  id: string;
+
+  @Column() guild_id: string;
+
+  @Column() channel_id: string;
+
+  @Column() user_id: string;
+
+  @Column() is_bot: boolean;
+
+  @Column("simple-json") data: ISavedMessageData;
+
+  @Column() posted_at: string;
+
+  @Column() deleted_at: string;
+}

src/index.ts

@@ -44,6 +44,7 @@ import { CensorPlugin } from "./plugins/Censor";
 import { PersistPlugin } from "./plugins/Persist";
 import { SpamPlugin } from "./plugins/Spam";
 import { TagsPlugin } from "./plugins/Tags";
+import { MessageSaverPlugin } from "./plugins/MessageSaver";
 
 // Run latest database migrations
 logger.info("Running database migrations");
@@ -57,6 +58,8 @@ connect().then(async conn => {
 
   const bot = new Knub(client, {
     plugins: {
+      messageSaver: MessageSaverPlugin,
+
       utility: UtilityPlugin,
       mod_actions: ModActionsPlugin,
       logs: LogsPlugin,
@@ -77,7 +80,7 @@ connect().then(async conn => {
         const plugins = guildConfig.plugins || {};
         const keys: string[] = Array.from(this.plugins.keys());
         return keys.filter(pluginName => {
-          return plugins[pluginName] && plugins[pluginName].enabled !== false;
+          return (plugins[pluginName] && plugins[pluginName].enabled !== false) || pluginName === "messageSaver";
         });
       },
 

src/plugins/MessageSaver.ts

@@ -0,0 +1,63 @@
+import { Plugin, decorators as d } from "knub";
+import { Message } from "eris";
+import { GuildSavedMessages } from "../data/GuildSavedMessages";
+import { ISavedMessageData } from "../data/entities/SavedMessage";
+import moment from "moment-timezone";
+
+export class MessageSaverPlugin extends Plugin {
+  protected messages: GuildSavedMessages;
+
+  onLoad() {
+    this.messages = GuildSavedMessages.getInstance(this.guildId);
+  }
+
+  protected msgToSavedMessageData(msg: Message): ISavedMessageData {
+    return {
+      attachments: msg.attachments,
+      author: {
+        username: msg.author.username,
+        discriminator: msg.author.discriminator
+      },
+      content: msg.content,
+      embeds: msg.embeds
+    };
+  }
+
+  @d.event("messageCreate", "guild", false)
+  async onMessageCreate(msg: Message) {
+    // Only save regular chat messages
+    if (msg.type !== 0) {
+      return;
+    }
+
+    const data: ISavedMessageData = this.msgToSavedMessageData(msg);
+    const postedAt = moment.utc(msg.timestamp, "x").format("YYYY-MM-DD HH:mm:ss.SSS");
+
+    await this.messages.create({
+      id: msg.id,
+      channel_id: msg.channel.id,
+      user_id: msg.author.id,
+      is_bot: msg.author.bot,
+      data,
+      posted_at: postedAt
+    });
+  }
+
+  @d.event("messageDelete", "guild", false)
+  async onMessageDelete(msg: Message) {
+    const savedMessage = await this.messages.find(msg.id);
+    if (!savedMessage) return;
+
+    await this.messages.markAsDeleted(msg.id);
+  }
+
+  @d.event("messageUpdate", "guild", false)
+  async onMessageUpdate(msg: Message) {
+    const savedMessage = await this.messages.find(msg.id);
+    if (!savedMessage) return;
+
+    const newData = this.msgToSavedMessageData(msg);
+
+    await this.messages.edit(msg.id, newData);
+  }
+}