debug: log query counts based on operation and table

backend/ormconfig.js

@@ -1,13 +1,20 @@
 const fs = require("fs");
 const path = require("path");
+const pkgUp = require("pkg-up");
+
+const closestPackageJson = pkgUp.sync();
+if (!closestPackageJson) {
+  throw new Error("Could not find project root from ormconfig.js");
+}
+const backendRoot = path.dirname(closestPackageJson);
 
 try {
-  fs.accessSync(path.resolve(__dirname, "bot.env"));
-  require("dotenv").config({ path: path.resolve(__dirname, "bot.env") });
+  fs.accessSync(path.resolve(backendRoot, "bot.env"));
+  require("dotenv").config({ path: path.resolve(backendRoot, "bot.env") });
 } catch {
   try {
-    fs.accessSync(path.resolve(__dirname, "api.env"));
-    require("dotenv").config({ path: path.resolve(__dirname, "api.env") });
+    fs.accessSync(path.resolve(backendRoot, "api.env"));
+    require("dotenv").config({ path: path.resolve(backendRoot, "api.env") });
   } catch {
     throw new Error("bot.env or api.env required");
   }
@@ -16,9 +23,9 @@ try {
 const moment = require("moment-timezone");
 moment.tz.setDefault("UTC");
 
-const entities = path.relative(process.cwd(), path.resolve(__dirname, "dist/backend/src/data/entities/*.js"));
-const migrations = path.relative(process.cwd(), path.resolve(__dirname, "dist/backend/src/migrations/*.js"));
-const migrationsDir = path.relative(process.cwd(), path.resolve(__dirname, "src/migrations"));
+const entities = path.relative(process.cwd(), path.resolve(backendRoot, "dist/backend/src/data/entities/*.js"));
+const migrations = path.relative(process.cwd(), path.resolve(backendRoot, "dist/backend/src/migrations/*.js"));
+const migrationsDir = path.relative(process.cwd(), path.resolve(backendRoot, "src/migrations"));
 
 module.exports = {
   type: "mysql",

backend/src/data/db.ts

@@ -1,5 +1,7 @@
 import { Connection, createConnection } from "typeorm";
 import { SimpleError } from "../SimpleError";
+import connectionOptions from "../../ormconfig";
+import { QueryLogger } from "./queryLogger";
 
 let connectionPromise: Promise<Connection>;
 
@@ -7,7 +9,11 @@ export let connection: Connection;
 
 export function connect() {
   if (!connectionPromise) {
-    connectionPromise = createConnection().then((newConnection) => {
+    connectionPromise = createConnection({
+      ...(connectionOptions as any),
+      logging: ["query", "error"],
+      logger: new QueryLogger(),
+    }).then((newConnection) => {
       // Verify the DB timezone is set to UTC
       return newConnection.query("SELECT TIMEDIFF(NOW(), UTC_TIMESTAMP) AS tz").then((r) => {
         if (r[0].tz !== "00:00:00") {

backend/src/index.ts

@@ -17,7 +17,7 @@ import { RecoverablePluginError } from "./RecoverablePluginError";
 import { SimpleError } from "./SimpleError";
 import { ZeppelinGlobalConfig, ZeppelinGuildConfig } from "./types";
 import { startUptimeCounter } from "./uptime";
-import { errorMessage, isDiscordAPIError, isDiscordHTTPError, SECONDS, sleep, successMessage } from "./utils";
+import { errorMessage, isDiscordAPIError, isDiscordHTTPError, MINUTES, SECONDS, sleep, successMessage } from "./utils";
 import { loadYamlSafely } from "./utils/loadYamlSafely";
 import { DecayingCounter } from "./utils/DecayingCounter";
 import { PluginNotLoadedError } from "knub/dist/plugins/PluginNotLoadedError";
@@ -35,6 +35,7 @@ import { setProfiler } from "./profiler";
 import { enableProfiling } from "./utils/easyProfiler";
 import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./data/loops/phishermanLoops";
 import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
+import { consumeQueryStats } from "./data/queryLogger";
 
 if (!process.env.KEY) {
   // tslint:disable-next-line:no-console
@@ -388,6 +389,17 @@ connect().then(async () => {
     lowestGlobalRemaining = Infinity;
   }, 15000);
 
+  setInterval(() => {
+    const queryStatsMap = consumeQueryStats();
+    const entries = Array.from(queryStatsMap.entries());
+    entries.sort((a, b) => b[1] - a[1]);
+    const topEntriesStr = entries
+      .slice(0, 5)
+      .map(([key, count]) => `${count}x ${key}`)
+      .join("\n");
+    console.log(`Top query entries in the past 5 minutes:\n${topEntriesStr}`);
+  }, 5 * MINUTES);
+
   bot.initialize();
   logger.info("Bot Initialized");
   logger.info("Logging in...");

backend/tsconfig.json

@@ -21,7 +21,11 @@
     "skipLibCheck": true,
     "strict": true,
     "strictPropertyInitialization": false,
-    "useUnknownInCatchVariables": false
+    "useUnknownInCatchVariables": false,
+    "allowJs": true
   },
-  "include": ["src/**/*.ts"]
+  "include": [
+    "src/**/*.ts",
+    "ormconfig.js"
+  ]
 }