Initial work on stats

backend/src/data/GuildStats.ts

@@ -0,0 +1,30 @@
+import { BaseGuildRepository } from "./BaseGuildRepository";
+import { connection } from "./db";
+import { getRepository, Repository } from "typeorm";
+import { StatValue } from "./entities/StatValue";
+
+export class GuildStats extends BaseGuildRepository {
+  private stats: Repository<StatValue>;
+
+  constructor(guildId) {
+    super(guildId);
+    this.stats = getRepository(StatValue);
+  }
+
+  async saveValue(source: string, key: string, value: number): Promise<void> {
+    await this.stats.insert({
+      guild_id: this.guildId,
+      source,
+      key,
+      value,
+    });
+  }
+
+  async deleteOldValues(source: string, cutoff: string): Promise<void> {
+    await this.stats
+      .createQueryBuilder()
+      .where("source = :source", { source })
+      .andWhere("created_at < :cutoff", { cutoff })
+      .delete();
+  }
+}

backend/src/data/entities/StatValue.ts

@@ -0,0 +1,20 @@
+import { Entity, Column, PrimaryColumn, CreateDateColumn } from "typeorm";
+
+@Entity("stats")
+export class StatValue {
+  @Column()
+  @PrimaryColumn()
+  id: string;
+
+  @Column()
+  guild_id: string;
+
+  @Column()
+  source: string;
+
+  @Column() key: string;
+
+  @Column() value: number;
+
+  @Column() created_at: string;
+}

backend/src/migrations/1575199835233-CreateStatsTable.ts

@@ -0,0 +1,59 @@
+import { MigrationInterface, QueryRunner, Table } from "typeorm";
+
+export class CreateStatsTable1575199835233 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.createTable(
+      new Table({
+        name: "stats",
+        columns: [
+          {
+            name: "id",
+            type: "bigint",
+            unsigned: true,
+            isPrimary: true,
+            generationStrategy: "increment",
+          },
+          {
+            name: "guild_id",
+            type: "bigint",
+            unsigned: true,
+          },
+          {
+            name: "source",
+            type: "varchar",
+            length: "64",
+            collation: "ascii_bin",
+          },
+          {
+            name: "key",
+            type: "varchar",
+            length: "64",
+            collation: "ascii_bin",
+          },
+          {
+            name: "value",
+            type: "integer",
+            unsigned: true,
+          },
+          {
+            name: "created_at",
+            type: "datetime",
+            default: "NOW()",
+          },
+        ],
+        indices: [
+          {
+            columnNames: ["guild_id", "source", "key"],
+          },
+          {
+            columnNames: ["created_at"],
+          },
+        ],
+      }),
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<any> {
+    await queryRunner.dropTable("stats");
+  }
+}

backend/src/plugins/Stats.ts

@@ -0,0 +1,155 @@
+import { ZeppelinPlugin } from "./ZeppelinPlugin";
+import * as t from "io-ts";
+import { convertDelayStringToMS, DAYS, HOURS, tAlphanumeric, tDateTime, tDeepPartial, tDelayString } from "../utils";
+import { IPluginOptions } from "knub";
+import moment from "moment-timezone";
+import { GuildStats } from "../data/GuildStats";
+import { Message } from "eris";
+import escapeStringRegexp from "escape-string-regexp";
+import { SavedMessage } from "../data/entities/SavedMessage";
+import { GuildSavedMessages } from "../data/GuildSavedMessages";
+
+const tBaseSource = t.type({
+  name: tAlphanumeric,
+  track: t.boolean,
+  retention_period: tDelayString,
+});
+
+const tMemberMessagesSource = t.intersection([
+  tBaseSource,
+  t.type({
+    type: t.literal("member_messages"),
+  }),
+]);
+type TMemberMessagesSource = t.TypeOf<typeof tMemberMessagesSource>;
+
+const tChannelMessagesSource = t.intersection([
+  tBaseSource,
+  t.type({
+    type: t.literal("channel_messages"),
+  }),
+]);
+type TChannelMessagesSource = t.TypeOf<typeof tChannelMessagesSource>;
+
+const tKeywordsSource = t.intersection([
+  tBaseSource,
+  t.type({
+    type: t.literal("keywords"),
+    keywords: t.array(t.string),
+  }),
+]);
+type TKeywordsSource = t.TypeOf<typeof tKeywordsSource>;
+
+const tSource = t.union([tMemberMessagesSource, tChannelMessagesSource, tKeywordsSource]);
+type TSource = t.TypeOf<typeof tSource>;
+
+const tConfigSchema = t.type({
+  sources: t.record(tAlphanumeric, tSource),
+});
+
+type TConfigSchema = t.TypeOf<typeof tConfigSchema>;
+const tPartialConfigSchema = tDeepPartial(tConfigSchema);
+
+const DEFAULT_RETENTION_PERIOD = "4w";
+
+export class StatsPlugin extends ZeppelinPlugin<TConfigSchema> {
+  public static pluginName = "stats";
+  public static configSchema = tConfigSchema;
+  public static showInDocs = false;
+
+  protected stats: GuildStats;
+  protected savedMessages: GuildSavedMessages;
+
+  private onMessageCreateFn;
+  private cleanStatsInterval;
+
+  public static getStaticDefaultOptions(): IPluginOptions<TConfigSchema> {
+    return {
+      config: {
+        sources: {},
+      },
+    };
+  }
+
+  protected static preprocessStaticConfig(config: t.TypeOf<typeof tPartialConfigSchema>) {
+    // TODO: Limit min period, min period start date
+
+    if (config.sources) {
+      for (const [key, source] of Object.entries(config.sources)) {
+        source.name = key;
+
+        if (source.track == null) {
+          source.track = true;
+        }
+
+        if (source.retention_period == null) {
+          source.retention_period = DEFAULT_RETENTION_PERIOD;
+        }
+      }
+    }
+
+    return config;
+  }
+
+  protected onLoad() {
+    this.stats = GuildStats.getGuildInstance(this.guildId);
+    this.savedMessages = GuildSavedMessages.getGuildInstance(this.guildId);
+
+    this.onMessageCreateFn = this.savedMessages.events.on("create", msg => this.onMessageCreate(msg));
+
+    this.cleanOldStats();
+    this.cleanStatsInterval = setInterval(() => this.cleanOldStats(), 1 * DAYS);
+  }
+
+  protected onUnload() {
+    this.savedMessages.events.off("create", this.onMessageCreateFn);
+    clearInterval(this.cleanStatsInterval);
+  }
+
+  protected async cleanOldStats() {
+    const config = this.getConfig();
+    for (const source of Object.values(config.sources)) {
+      const cutoffMS = convertDelayStringToMS(source.retention_period);
+      const cutoff = moment()
+        .subtract(cutoffMS, "ms")
+        .format("YYYY-MM-DD HH:mm:ss");
+      await this.stats.deleteOldValues(source.name, cutoff);
+    }
+  }
+
+  protected saveMemberMessagesStats(source: TMemberMessagesSource, msg: SavedMessage) {
+    this.stats.saveValue(source.name, msg.user_id, 1);
+  }
+
+  protected saveChannelMessagesStats(source: TChannelMessagesSource, msg: SavedMessage) {
+    this.stats.saveValue(source.name, msg.channel_id, 1);
+  }
+
+  protected saveKeywordsStats(source: TKeywordsSource, msg: SavedMessage) {
+    const content = msg.data.content;
+    if (!content) return;
+
+    for (const keyword of source.keywords) {
+      const regex = new RegExp(`\\b${escapeStringRegexp(keyword)}\\b`, "i");
+      if (content.match(regex)) {
+        this.stats.saveValue(source.name, "keyword", 1);
+        break;
+      }
+    }
+  }
+
+  onMessageCreate(msg: SavedMessage) {
+    const config = this.getConfigForMemberIdAndChannelId(msg.user_id, msg.channel_id);
+    for (const source of Object.values(config.sources)) {
+      if (!source.track) continue;
+
+      if (source.type === "member_messages") {
+        this.saveMemberMessagesStats(source, msg);
+      } else if (source.type === "channel_messages") {
+        this.saveChannelMessagesStats(source, msg);
+      } else if (source.type === "keywords") {
+        this.saveKeywordsStats(source, msg);
+      }
+    }
+  }
+}

backend/src/utils.test.ts

@@ -1,21 +1,47 @@
-import { getUrlsInString } from "./utils";
+import { convertDelayStringToMS, convertMSToDelayString, getUrlsInString } from "./utils";
 
 import test from "ava";
 
-test("Detects full links", t => {
+test("getUrlsInString(): detects full links", t => {
   const urls = getUrlsInString("foo https://google.com/ bar");
   t.is(urls.length, 1);
   t.is(urls[0].hostname, "google.com");
 });
 
-test("Detects partial links", t => {
+test("getUrlsInString(): detects partial links", t => {
   const urls = getUrlsInString("foo google.com bar");
   t.is(urls.length, 1);
   t.is(urls[0].hostname, "google.com");
 });
 
-test("Detects subdomains", t => {
+test("getUrlsInString(): detects subdomains", t => {
   const urls = getUrlsInString("foo photos.google.com bar");
   t.is(urls.length, 1);
   t.is(urls[0].hostname, "photos.google.com");
 });
+
+test("delay strings: basic support", t => {
+  const delayString = "2w4d7h32m17s";
+  const expected = 1_582_337_000;
+  t.is(convertDelayStringToMS(delayString), expected);
+});
+
+test("delay strings: default unit (minutes)", t => {
+  t.is(convertDelayStringToMS("10"), 10 * 60 * 1000);
+});
+
+test("delay strings: custom default unit", t => {
+  t.is(convertDelayStringToMS("10", "s"), 10 * 1000);
+});
+
+test("delay strings: reverse conversion", t => {
+  const ms = 1_582_337_020;
+  const expected = "2w4d7h32m17s20x";
+  t.is(convertMSToDelayString(ms), expected);
+});
+
+test("delay strings: reverse conversion (conservative)", t => {
+  const ms = 1_209_600_000;
+  const expected = "2w";
+  t.is(convertMSToDelayString(ms), expected);
+});

backend/src/utils.ts

@@ -29,6 +29,9 @@ import tmp from "tmp";
 import { logger, waitForReaction } from "knub";
 import { SavedMessage } from "./data/entities/SavedMessage";
 import { decodeAndValidateStrict, StrictValidationError } from "./validatorUtils";
+import { either } from "fp-ts/lib/Either";
+import safeRegex from "safe-regex";
+import moment from "moment-timezone";
 
 const delayStringMultipliers = {
   w: 1000 * 60 * 60 * 24 * 7,
@@ -36,6 +39,7 @@ const delayStringMultipliers = {
   h: 1000 * 60 * 60,
   m: 1000 * 60,
   s: 1000,
+  x: 1,
 };
 
 export const MS = 1;
@@ -184,6 +188,40 @@ export function dropPropertiesByName(obj, propName) {
   }
 }
 
+export const tAlphanumeric = new t.Type<string, string>(
+  "tAlphanumeric",
+  (s): s is string => typeof s === "string",
+  (from, to) =>
+    either.chain(t.string.validate(from, to), s => {
+      return s.match(/\W/) ? t.failure(from, to, "String must be alphanumeric") : t.success(s);
+    }),
+  s => s,
+);
+
+export const tDateTime = new t.Type<string, string>(
+  "tDateTime",
+  (s): s is string => typeof s === "string",
+  (from, to) =>
+    either.chain(t.string.validate(from, to), s => {
+      const parsed =
+        s.length === 10 ? moment(s, "YYYY-MM-DD") : s.length === 19 ? moment(s, "YYYY-MM-DD HH:mm:ss") : null;
+
+      return parsed && parsed.isValid() ? t.success(s) : t.failure(from, to, "Invalid datetime");
+    }),
+  s => s,
+);
+
+export const tDelayString = new t.Type<string, string>(
+  "tDelayString",
+  (s): s is string => typeof s === "string",
+  (from, to) =>
+    either.chain(t.string.validate(from, to), s => {
+      const ms = convertDelayStringToMS(s);
+      return ms === null ? t.failure(from, to, "Invalid delay string") : t.success(s);
+    }),
+  s => s,
+);
+
 /**
  * Turns a "delay string" such as "1h30m" to milliseconds
  */
@@ -208,6 +246,21 @@ export function convertDelayStringToMS(str, defaultUnit = "m"): number {
   return ms;
 }
 
+export function convertMSToDelayString(ms: number): string {
+  let result = "";
+  let remaining = ms;
+  for (const [abbr, multiplier] of Object.entries(delayStringMultipliers)) {
+    if (multiplier <= remaining) {
+      const amount = Math.floor(remaining / multiplier);
+      result += `${amount}${abbr}`;
+      remaining -= amount * multiplier;
+    }
+
+    if (remaining === 0) break;
+  }
+  return result;
+}
+
 export function successMessage(str) {
   return `<:zep_check:650361014180904971> ${str}`;
 }