Prepare to migrate usernames to a new table

src/index.ts

@@ -95,7 +95,7 @@ import { startUptimeCounter } from "./uptime";
 // Run latest database migrations
 logger.info("Running database migrations");
 connect().then(async conn => {
-  await conn.runMigrations();
+  // await conn.runMigrations();
 
   const client = new Client(`Bot ${process.env.TOKEN}`, {
     getAllUsers: true,

src/migrations/1556908589679-CreateUsernameHistoryTable.ts

@@ -0,0 +1,46 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreateUsernameHistoryTable1556908589679 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.createTable(
+      new Table({
+        name: "username_history",
+        columns: [
+          {
+            name: "id",
+            type: "int",
+            unsigned: true,
+            isGenerated: true,
+            generationStrategy: "increment",
+            isPrimary: true,
+          },
+          {
+            name: "user_id",
+            type: "bigint",
+            unsigned: true,
+          },
+          {
+            name: "username",
+            type: "varchar",
+            length: "160",
+            isNullable: true,
+          },
+          {
+            name: "timestamp",
+            type: "datetime",
+            default: "CURRENT_TIMESTAMP",
+          },
+        ],
+        indices: [
+          {
+            columnNames: ["user_id"],
+          },
+        ],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropTable("username_history", true);
+  }
+}

src/migrations/1556909512501-MigrateUsernamesToNewHistoryTable.ts

@@ -0,0 +1,63 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+const BATCH_SIZE = 200;
+
+export class MigrateUsernamesToNewHistoryTable1556909512501 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    // Start by ending the migration transaction because this is gonna be a looooooooot of data
+    await queryRunner.query("COMMIT");
+
+    const migratedUsernames = new Set();
+
+    const migrateNextBatch = (): Promise<{ finished: boolean; migrated?: number }> => {
+      return new Promise(async resolve => {
+        const toInsert = [];
+        const toDelete = [];
+
+        const stream = await queryRunner.stream(
+          `SELECT * FROM name_history WHERE type=1 ORDER BY timestamp ASC LIMIT ${BATCH_SIZE}`,
+        );
+        stream.on("result", row => {
+          const key = `${row.user_id}-${row.value}`;
+          if (migratedUsernames.has(key)) return;
+          migratedUsernames.add(key);
+
+          toInsert.push([row.user_id, row.value, row.timestamp]);
+          toDelete.push(row.id);
+        });
+        stream.on("end", async () => {
+          if (toInsert.length) {
+            await queryRunner.query("START TRANSACTION");
+            await queryRunner.query(
+              "INSERT INTO username_history (user_id, username, timestamp) VALUES " +
+                Array.from({ length: toInsert.length }, () => "(?, ?, ?)").join(","),
+              toInsert.flat(),
+            );
+            await queryRunner.query(
+              "DELETE FROM name_history WHERE id IN (" + Array.from("?".repeat(toDelete.length)).join(", ") + ")",
+              toDelete,
+            );
+            await queryRunner.query("COMMIT");
+
+            resolve({ finished: false, migrated: toInsert.length });
+          } else {
+            resolve({ finished: true });
+          }
+        });
+      });
+    };
+
+    while (true) {
+      const result = await migrateNextBatch();
+      if (result.finished) {
+        break;
+      } else {
+        console.log(`Migrated ${result.migrated} usernames`);
+      }
+    }
+
+    await queryRunner.query("START TRANSACTION");
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {}
+}

src/plugins/NameHistory.ts

@@ -61,13 +61,13 @@ export class NameHistoryPlugin extends ZeppelinPlugin<INameHistoryPluginConfig>
     createChunkedMessage(msg.channel, message);
   }
 
-  @d.event("userUpdate", null, false)
+  // @d.event("userUpdate", null, false)
-  async onUserUpdate(user: User, oldUser: { username: string; discriminator: string; avatar: string }) {
+  // async onUserUpdate(user: User, oldUser: { username: string; discriminator: string; avatar: string }) {
-    if (user.username !== oldUser.username || user.discriminator !== oldUser.discriminator) {
+  //   if (user.username !== oldUser.username || user.discriminator !== oldUser.discriminator) {
-      const newUsername = `${user.username}#${user.discriminator}`;
+  //     const newUsername = `${user.username}#${user.discriminator}`;
-      await this.nameHistory.addEntry(user.id, NameHistoryEntryTypes.Username, newUsername);
+  //     await this.nameHistory.addEntry(user.id, NameHistoryEntryTypes.Username, newUsername);
-    }
+  //   }
-  }
+  // }
 
   @d.event("guildMemberUpdate")
   async onGuildMemberUpdate(_, member: Member) {
@@ -77,12 +77,12 @@ export class NameHistoryPlugin extends ZeppelinPlugin<INameHistoryPluginConfig>
     }
   }
 
-  @d.event("guildMemberAdd")
+  // @d.event("guildMemberAdd")
-  async onGuildMemberAdd(_, member: Member) {
+  // async onGuildMemberAdd(_, member: Member) {
-    const latestEntry = await this.nameHistory.getLastEntryByType(member.id, NameHistoryEntryTypes.Username);
+  //   const latestEntry = await this.nameHistory.getLastEntryByType(member.id, NameHistoryEntryTypes.Username);
-    const username = `${member.user.username}#${member.user.discriminator}`;
+  //   const username = `${member.user.username}#${member.user.discriminator}`;
-    if (!latestEntry || latestEntry.value !== username) {
+  //   if (!latestEntry || latestEntry.value !== username) {
-      await this.nameHistory.addEntry(member.id, NameHistoryEntryTypes.Username, username);
+  //     await this.nameHistory.addEntry(member.id, NameHistoryEntryTypes.Username, username);
-    }
+  //   }
-  }
+  // }
 }