From 56fb432c7c3bf12b29b783aa259691b219f08459 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 1 Dec 2019 15:57:35 +0200 Subject: [PATCH] Initial work on stats --- backend/src/data/GuildStats.ts | 30 ++++ backend/src/data/entities/StatValue.ts | 20 +++ .../1575199835233-CreateStatsTable.ts | 59 +++++++ backend/src/plugins/Stats.ts | 155 ++++++++++++++++++ backend/src/utils.test.ts | 34 +++- backend/src/utils.ts | 53 ++++++ 6 files changed, 347 insertions(+), 4 deletions(-) create mode 100644 backend/src/data/GuildStats.ts create mode 100644 backend/src/data/entities/StatValue.ts create mode 100644 backend/src/migrations/1575199835233-CreateStatsTable.ts create mode 100644 backend/src/plugins/Stats.ts diff --git a/backend/src/data/GuildStats.ts b/backend/src/data/GuildStats.ts new file mode 100644 index 00000000..46c7497b --- /dev/null +++ b/backend/src/data/GuildStats.ts @@ -0,0 +1,30 @@ +import { BaseGuildRepository } from "./BaseGuildRepository"; +import { connection } from "./db"; +import { getRepository, Repository } from "typeorm"; +import { StatValue } from "./entities/StatValue"; + +export class GuildStats extends BaseGuildRepository { + private stats: Repository; + + constructor(guildId) { + super(guildId); + this.stats = getRepository(StatValue); + } + + async saveValue(source: string, key: string, value: number): Promise { + await this.stats.insert({ + guild_id: this.guildId, + source, + key, + value, + }); + } + + async deleteOldValues(source: string, cutoff: string): Promise { + await this.stats + .createQueryBuilder() + .where("source = :source", { source }) + .andWhere("created_at < :cutoff", { cutoff }) + .delete(); + } +} diff --git a/backend/src/data/entities/StatValue.ts b/backend/src/data/entities/StatValue.ts new file mode 100644 index 00000000..78978a28 --- /dev/null +++ b/backend/src/data/entities/StatValue.ts @@ -0,0 +1,20 @@ +import { Entity, Column, PrimaryColumn, CreateDateColumn } from "typeorm"; + +@Entity("stats") +export class StatValue { + @Column() + @PrimaryColumn() + id: string; + + @Column() + guild_id: string; + + @Column() + source: string; + + @Column() key: string; + + @Column() value: number; + + @Column() created_at: string; +} diff --git a/backend/src/migrations/1575199835233-CreateStatsTable.ts b/backend/src/migrations/1575199835233-CreateStatsTable.ts new file mode 100644 index 00000000..d6c6a8d7 --- /dev/null +++ b/backend/src/migrations/1575199835233-CreateStatsTable.ts @@ -0,0 +1,59 @@ +import { MigrationInterface, QueryRunner, Table } from "typeorm"; + +export class CreateStatsTable1575199835233 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.createTable( + new Table({ + name: "stats", + columns: [ + { + name: "id", + type: "bigint", + unsigned: true, + isPrimary: true, + generationStrategy: "increment", + }, + { + name: "guild_id", + type: "bigint", + unsigned: true, + }, + { + name: "source", + type: "varchar", + length: "64", + collation: "ascii_bin", + }, + { + name: "key", + type: "varchar", + length: "64", + collation: "ascii_bin", + }, + { + name: "value", + type: "integer", + unsigned: true, + }, + { + name: "created_at", + type: "datetime", + default: "NOW()", + }, + ], + indices: [ + { + columnNames: ["guild_id", "source", "key"], + }, + { + columnNames: ["created_at"], + }, + ], + }), + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.dropTable("stats"); + } +} diff --git a/backend/src/plugins/Stats.ts b/backend/src/plugins/Stats.ts new file mode 100644 index 00000000..63c12856 --- /dev/null +++ b/backend/src/plugins/Stats.ts @@ -0,0 +1,155 @@ +import { ZeppelinPlugin } from "./ZeppelinPlugin"; +import * as t from "io-ts"; +import { convertDelayStringToMS, DAYS, HOURS, tAlphanumeric, tDateTime, tDeepPartial, tDelayString } from "../utils"; +import { IPluginOptions } from "knub"; +import moment from "moment-timezone"; +import { GuildStats } from "../data/GuildStats"; +import { Message } from "eris"; +import escapeStringRegexp from "escape-string-regexp"; +import { SavedMessage } from "../data/entities/SavedMessage"; +import { GuildSavedMessages } from "../data/GuildSavedMessages"; + +const tBaseSource = t.type({ + name: tAlphanumeric, + track: t.boolean, + retention_period: tDelayString, +}); + +const tMemberMessagesSource = t.intersection([ + tBaseSource, + t.type({ + type: t.literal("member_messages"), + }), +]); +type TMemberMessagesSource = t.TypeOf; + +const tChannelMessagesSource = t.intersection([ + tBaseSource, + t.type({ + type: t.literal("channel_messages"), + }), +]); +type TChannelMessagesSource = t.TypeOf; + +const tKeywordsSource = t.intersection([ + tBaseSource, + t.type({ + type: t.literal("keywords"), + keywords: t.array(t.string), + }), +]); +type TKeywordsSource = t.TypeOf; + +const tSource = t.union([tMemberMessagesSource, tChannelMessagesSource, tKeywordsSource]); +type TSource = t.TypeOf; + +const tConfigSchema = t.type({ + sources: t.record(tAlphanumeric, tSource), +}); + +type TConfigSchema = t.TypeOf; +const tPartialConfigSchema = tDeepPartial(tConfigSchema); + +const DEFAULT_RETENTION_PERIOD = "4w"; + +export class StatsPlugin extends ZeppelinPlugin { + public static pluginName = "stats"; + public static configSchema = tConfigSchema; + public static showInDocs = false; + + protected stats: GuildStats; + protected savedMessages: GuildSavedMessages; + + private onMessageCreateFn; + private cleanStatsInterval; + + public static getStaticDefaultOptions(): IPluginOptions { + return { + config: { + sources: {}, + }, + }; + } + + protected static preprocessStaticConfig(config: t.TypeOf) { + // TODO: Limit min period, min period start date + + if (config.sources) { + for (const [key, source] of Object.entries(config.sources)) { + source.name = key; + + if (source.track == null) { + source.track = true; + } + + if (source.retention_period == null) { + source.retention_period = DEFAULT_RETENTION_PERIOD; + } + } + } + + return config; + } + + protected onLoad() { + this.stats = GuildStats.getGuildInstance(this.guildId); + this.savedMessages = GuildSavedMessages.getGuildInstance(this.guildId); + + this.onMessageCreateFn = this.savedMessages.events.on("create", msg => this.onMessageCreate(msg)); + + this.cleanOldStats(); + this.cleanStatsInterval = setInterval(() => this.cleanOldStats(), 1 * DAYS); + } + + protected onUnload() { + this.savedMessages.events.off("create", this.onMessageCreateFn); + clearInterval(this.cleanStatsInterval); + } + + protected async cleanOldStats() { + const config = this.getConfig(); + for (const source of Object.values(config.sources)) { + const cutoffMS = convertDelayStringToMS(source.retention_period); + const cutoff = moment() + .subtract(cutoffMS, "ms") + .format("YYYY-MM-DD HH:mm:ss"); + await this.stats.deleteOldValues(source.name, cutoff); + } + } + + protected saveMemberMessagesStats(source: TMemberMessagesSource, msg: SavedMessage) { + this.stats.saveValue(source.name, msg.user_id, 1); + } + + protected saveChannelMessagesStats(source: TChannelMessagesSource, msg: SavedMessage) { + this.stats.saveValue(source.name, msg.channel_id, 1); + } + + protected saveKeywordsStats(source: TKeywordsSource, msg: SavedMessage) { + const content = msg.data.content; + if (!content) return; + + for (const keyword of source.keywords) { + const regex = new RegExp(`\\b${escapeStringRegexp(keyword)}\\b`, "i"); + if (content.match(regex)) { + this.stats.saveValue(source.name, "keyword", 1); + break; + } + } + } + + onMessageCreate(msg: SavedMessage) { + const config = this.getConfigForMemberIdAndChannelId(msg.user_id, msg.channel_id); + for (const source of Object.values(config.sources)) { + if (!source.track) continue; + + if (source.type === "member_messages") { + this.saveMemberMessagesStats(source, msg); + } else if (source.type === "channel_messages") { + this.saveChannelMessagesStats(source, msg); + } else if (source.type === "keywords") { + this.saveKeywordsStats(source, msg); + } + } + } +} diff --git a/backend/src/utils.test.ts b/backend/src/utils.test.ts index cc0eca24..345c32a3 100644 --- a/backend/src/utils.test.ts +++ b/backend/src/utils.test.ts @@ -1,21 +1,47 @@ -import { getUrlsInString } from "./utils"; +import { convertDelayStringToMS, convertMSToDelayString, getUrlsInString } from "./utils"; import test from "ava"; -test("Detects full links", t => { +test("getUrlsInString(): detects full links", t => { const urls = getUrlsInString("foo https://google.com/ bar"); t.is(urls.length, 1); t.is(urls[0].hostname, "google.com"); }); -test("Detects partial links", t => { +test("getUrlsInString(): detects partial links", t => { const urls = getUrlsInString("foo google.com bar"); t.is(urls.length, 1); t.is(urls[0].hostname, "google.com"); }); -test("Detects subdomains", t => { +test("getUrlsInString(): detects subdomains", t => { const urls = getUrlsInString("foo photos.google.com bar"); t.is(urls.length, 1); t.is(urls[0].hostname, "photos.google.com"); }); + +test("delay strings: basic support", t => { + const delayString = "2w4d7h32m17s"; + const expected = 1_582_337_000; + t.is(convertDelayStringToMS(delayString), expected); +}); + +test("delay strings: default unit (minutes)", t => { + t.is(convertDelayStringToMS("10"), 10 * 60 * 1000); +}); + +test("delay strings: custom default unit", t => { + t.is(convertDelayStringToMS("10", "s"), 10 * 1000); +}); + +test("delay strings: reverse conversion", t => { + const ms = 1_582_337_020; + const expected = "2w4d7h32m17s20x"; + t.is(convertMSToDelayString(ms), expected); +}); + +test("delay strings: reverse conversion (conservative)", t => { + const ms = 1_209_600_000; + const expected = "2w"; + t.is(convertMSToDelayString(ms), expected); +}); diff --git a/backend/src/utils.ts b/backend/src/utils.ts index 139f9d18..5fa40b93 100644 --- a/backend/src/utils.ts +++ b/backend/src/utils.ts @@ -29,6 +29,9 @@ import tmp from "tmp"; import { logger, waitForReaction } from "knub"; import { SavedMessage } from "./data/entities/SavedMessage"; import { decodeAndValidateStrict, StrictValidationError } from "./validatorUtils"; +import { either } from "fp-ts/lib/Either"; +import safeRegex from "safe-regex"; +import moment from "moment-timezone"; const delayStringMultipliers = { w: 1000 * 60 * 60 * 24 * 7, @@ -36,6 +39,7 @@ const delayStringMultipliers = { h: 1000 * 60 * 60, m: 1000 * 60, s: 1000, + x: 1, }; export const MS = 1; @@ -184,6 +188,40 @@ export function dropPropertiesByName(obj, propName) { } } +export const tAlphanumeric = new t.Type( + "tAlphanumeric", + (s): s is string => typeof s === "string", + (from, to) => + either.chain(t.string.validate(from, to), s => { + return s.match(/\W/) ? t.failure(from, to, "String must be alphanumeric") : t.success(s); + }), + s => s, +); + +export const tDateTime = new t.Type( + "tDateTime", + (s): s is string => typeof s === "string", + (from, to) => + either.chain(t.string.validate(from, to), s => { + const parsed = + s.length === 10 ? moment(s, "YYYY-MM-DD") : s.length === 19 ? moment(s, "YYYY-MM-DD HH:mm:ss") : null; + + return parsed && parsed.isValid() ? t.success(s) : t.failure(from, to, "Invalid datetime"); + }), + s => s, +); + +export const tDelayString = new t.Type( + "tDelayString", + (s): s is string => typeof s === "string", + (from, to) => + either.chain(t.string.validate(from, to), s => { + const ms = convertDelayStringToMS(s); + return ms === null ? t.failure(from, to, "Invalid delay string") : t.success(s); + }), + s => s, +); + /** * Turns a "delay string" such as "1h30m" to milliseconds */ @@ -208,6 +246,21 @@ export function convertDelayStringToMS(str, defaultUnit = "m"): number { return ms; } +export function convertMSToDelayString(ms: number): string { + let result = ""; + let remaining = ms; + for (const [abbr, multiplier] of Object.entries(delayStringMultipliers)) { + if (multiplier <= remaining) { + const amount = Math.floor(remaining / multiplier); + result += `${amount}${abbr}`; + remaining -= amount * multiplier; + } + + if (remaining === 0) break; + } + return result; +} + export function successMessage(str) { return `<:zep_check:650361014180904971> ${str}`; }