Add adhoc REST call debug stats

backend/package.json

@@ -7,11 +7,11 @@
     "watch": "cross-env NODE_ENV=development tsc-watch --onSuccess \"node start-dev.js\"",
     "watch-yaml-parse-test": "cross-env NODE_ENV=development tsc-watch --onSuccess \"node dist/backend/src/yamlParseTest.js\"",
     "build": "rimraf dist && tsc",
-    "start-bot-dev": "cross-env NODE_ENV=development node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --inspect=0.0.0.0:9229 dist/backend/src/index.js",
-    "start-bot-prod": "cross-env NODE_ENV=production node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps dist/backend/src/index.js",
+    "start-bot-dev": "cross-env NODE_ENV=development node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9229 dist/backend/src/index.js",
+    "start-bot-prod": "cross-env NODE_ENV=production node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --stack-trace-limit=30 dist/backend/src/index.js",
     "watch-bot": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-bot-dev\"",
-    "start-api-dev": "cross-env NODE_ENV=development node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
-    "start-api-prod": "cross-env NODE_ENV=production node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps dist/backend/src/api/index.js",
+    "start-api-dev": "cross-env NODE_ENV=development node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
+    "start-api-prod": "cross-env NODE_ENV=production node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --stack-trace-limit=30 dist/backend/src/api/index.js",
     "watch-api": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-api-dev\"",
     "typeorm": "node -r ./register-tsconfig-paths.js ./node_modules/typeorm/cli.js",
     "migrate-prod": "npm run typeorm -- migration:run",

backend/src/index.ts

@@ -21,6 +21,7 @@ import { errorMessage, isDiscordAPIError, isDiscordHTTPError, SECONDS, successMe
 import { loadYamlSafely } from "./utils/loadYamlSafely";
 import { DecayingCounter } from "./utils/DecayingCounter";
 import { PluginNotLoadedError } from "knub/dist/plugins/PluginNotLoadedError";
+import { logRestCall } from "./restCallStats";
 
 if (!process.env.KEY) {
   // tslint:disable-next-line:no-console
@@ -156,6 +157,15 @@ moment.tz.setDefault("UTC");
 
 logger.info("Connecting to database");
 connect().then(async () => {
+  const RequestHandler = require("discord.js/src/rest/RequestHandler.js");
+  const originalPush = RequestHandler.prototype.push;
+  // tslint:disable-next-line:only-arrow-functions
+  RequestHandler.prototype.push = function (...args) {
+    const request = args[0];
+    logRestCall(request.method, request.path);
+    return originalPush.call(this, ...args);
+  };
+
   const client = new Client({
     partials: ["USER", "CHANNEL", "GUILD_MEMBER", "MESSAGE", "REACTION"],
 

backend/src/plugins/BotControl/BotControlPlugin.ts

@@ -18,9 +18,10 @@ import { ReloadServerCmd } from "./commands/ReloadServerCmd";
 import { RemoveDashboardUserCmd } from "./commands/RemoveDashboardUserCmd";
 import { ServersCmd } from "./commands/ServersCmd";
 import { BotControlPluginType, ConfigSchema } from "./types";
-import { PerformanceCmd } from "./commands/PerformanceCmd";
+import { PluginPerformanceCmd } from "./commands/PluginPerformanceCmd";
 import { AddServerFromInviteCmd } from "./commands/AddServerFromInviteCmd";
 import { ChannelToServerCmd } from "./commands/ChannelToServerCmd";
+import { RestPerformanceCmd } from "./commands/RestPerformanceCmd";
 
 const defaultOptions = {
   config: {
@@ -51,7 +52,8 @@ export const BotControlPlugin = zeppelinGlobalPlugin<BotControlPluginType>()({
     ListDashboardUsersCmd,
     ListDashboardPermsCmd,
     EligibleCmd,
-    PerformanceCmd,
+    PluginPerformanceCmd,
+    RestPerformanceCmd,
     AddServerFromInviteCmd,
     ChannelToServerCmd,
   ],

backend/src/plugins/BotControl/commands/PluginPerformanceCmd.ts

@@ -4,8 +4,8 @@ import { sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { createChunkedMessage, formatNumber, resolveInvite, sorter, verboseUserMention } from "../../../utils";
 import { botControlCmd } from "../types";
 
-export const PerformanceCmd = botControlCmd({
-  trigger: ["performance"],
+export const PluginPerformanceCmd = botControlCmd({
+  trigger: ["plugin_performance"],
   permission: "can_performance",
 
   signature: {},

backend/src/plugins/BotControl/commands/RestPerformanceCmd.ts

@@ -0,0 +1,24 @@
+import { TextChannel } from "discord.js";
+import { commandTypeHelpers as ct } from "../../../commandTypes";
+import { sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { createChunkedMessage, formatNumber, resolveInvite, sorter, verboseUserMention } from "../../../utils";
+import { botControlCmd } from "../types";
+import { getTopRestCallStats } from "../../../restCallStats";
+
+const leadingPathRegex = /(?<=\().+\/backend\//g;
+
+export const RestPerformanceCmd = botControlCmd({
+  trigger: ["rest_performance"],
+  permission: "can_performance",
+
+  signature: {},
+
+  async run({ pluginData, message: msg, args }) {
+    const stats = getTopRestCallStats(5);
+    const formatted = stats.map((callStats) => {
+      const cleanSource = callStats.source.replace(leadingPathRegex, "");
+      return `**${callStats.count} calls**\n${callStats.method.toUpperCase()} ${callStats.path}\n${cleanSource}`;
+    });
+    createChunkedMessage(msg.channel as TextChannel, formatted.join("\n"));
+  },
+});

backend/src/restCallStats.ts

@@ -0,0 +1,35 @@
+import { sorter } from "./utils";
+
+Error.stackTraceLimit = Infinity;
+
+type CallStats = { method: string; path: string; source: string; count: number };
+const restCallStats: Map<string, CallStats> = new Map();
+
+const looseSnowflakeRegex = /\d{15,}/g;
+const queryParamsRegex = /\?.*$/g;
+
+export function logRestCall(method: string, path: string) {
+  const anonymizedPath = path.replace(looseSnowflakeRegex, "0000").replace(queryParamsRegex, "");
+  const stackLines = (new Error().stack || "").split("\n").slice(10); // Remove initial fluff
+  const firstSrcLine = stackLines.findIndex((line) => line.includes("/backend/src"));
+  const source = stackLines
+    .slice(firstSrcLine !== -1 ? firstSrcLine : -5)
+    .filter((l) => !l.includes("processTicksAndRejections"))
+    .join("\n");
+  const key = `${method}|${anonymizedPath}|${source}`;
+  if (!restCallStats.has(key)) {
+    restCallStats.set(key, {
+      method,
+      path: anonymizedPath,
+      source,
+      count: 0,
+    });
+  }
+  restCallStats.get(key)!.count++;
+}
+
+export function getTopRestCallStats(count: number): CallStats[] {
+  const stats = Array.from(restCallStats.values());
+  stats.sort(sorter("count", "DESC"));
+  return stats.slice(0, count);
+}