StyAndCo/zappyzep
3
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add username/nickname history retention periods

Browse source
This commit is contained in:
Dragory 2020-06-02 00:26:06 +03:00
parent a6e650810c
commit de71520747
No known key found for this signature in database
GPG key ID: 5F387BA66DF8AAC1
8 changed files with 253 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
backend/src/data/GuildNicknameHistory.ts
View file

@ -1,7 +1,22 @@
import { BaseGuildRepository } from "./BaseGuildRepository"; import { BaseGuildRepository } from "./BaseGuildRepository";
import { getRepository, Repository } from "typeorm"; import { getRepository, In, Repository } from "typeorm";
import { NicknameHistoryEntry } from "./entities/NicknameHistoryEntry"; import { NicknameHistoryEntry } from "./entities/NicknameHistoryEntry";
import { sorter } from "../utils"; import { MINUTES, SECONDS, sorter } from "../utils";
import { MAX_USERNAME_ENTRIES_PER_USER } from "./UsernameHistory";
import { isAPI } from "../globals";
import { cleanupNicknames } from "./cleanup/nicknames";
if (!isAPI()) {
const CLEANUP_INTERVAL = 5 * MINUTES;
async function cleanup() {
await cleanupNicknames();
setTimeout(cleanup, CLEANUP_INTERVAL);
}
// Start first cleanup 30 seconds after startup
setTimeout(cleanup, 30 * SECONDS);
}
export const MAX_NICKNAME_ENTRIES_PER_USER = 10; export const MAX_NICKNAME_ENTRIES_PER_USER = 10;
@ -44,25 +59,20 @@ export class GuildNicknameHistory extends BaseGuildRepository {
nickname, nickname,
}); });
// Cleanup (leave only the last MAX_NICKNAME_ENTRIES_PER_USER entries) // Cleanup (leave only the last MAX_USERNAME_ENTRIES_PER_USER entries)
const lastEntries = await this.getByUserId(userId); const toDelete = await this.nicknameHistory
if (lastEntries.length > MAX_NICKNAME_ENTRIES_PER_USER) { .createQueryBuilder()
const earliestEntry = lastEntries .where("guild_id = :guildId", { guildId: this.guildId })
.sort(sorter("timestamp", "DESC")) .andWhere("user_id = :userId", { userId })
.slice(0, 10) .orderBy("id", "DESC")
.reduce((earliest, entry) => { .skip(MAX_USERNAME_ENTRIES_PER_USER)
if (earliest == null) return entry; .take(99_999)
if (entry.id < earliest.id) return entry; .getMany();
return earliest;
}, null);
this.nicknameHistory if (toDelete.length > 0) {
.createQueryBuilder() await this.nicknameHistory.delete({
.where("guild_id = :guildId", { guildId: this.guildId }) id: In(toDelete.map(v => v.id)),
.andWhere("user_id = :userId", { userId }) });
.andWhere("id < :id", { id: earliestEntry.id })
.delete()
.execute();
} }
} }
} }

68
backend/src/data/GuildSavedMessages.ts
View file

@ -7,68 +7,16 @@ import moment from "moment-timezone";
import { DAYS, DBDateFormat, MINUTES, SECONDS } from "../utils"; import { DAYS, DBDateFormat, MINUTES, SECONDS } from "../utils";
import { isAPI } from "../globals"; import { isAPI } from "../globals";
import { connection } from "./db"; import { connection } from "./db";
import { cleanupMessages } from "./cleanup/messages";
const CLEANUP_INTERVAL = 5 * MINUTES;
/**
* How long message edits, deletions, etc. will include the original message content.
* This is very heavy storage-wise, so keeping it as low as possible is ideal.
*/
const RETENTION_PERIOD = 1 * DAYS;
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
const CLEAN_PER_LOOP = 250;
async function cleanup() {
const repo = getRepository(SavedMessage);
const deletedAtThreshold = moment()
.subtract(CLEANUP_INTERVAL, "ms")
.format(DBDateFormat);
const postedAtThreshold = moment()
.subtract(RETENTION_PERIOD, "ms")
.format(DBDateFormat);
const botPostedAtThreshold = moment()
.subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms")
.format(DBDateFormat);
// SELECT + DELETE messages in batches
// This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
// when a message was being inserted at the same time
let rows;
do {
rows = await connection.query(
`
SELECT id
FROM messages
WHERE (
deleted_at IS NOT NULL
AND deleted_at <= ?
)
OR (
posted_at <= ?
AND is_permanent = 0
)
OR (
is_bot = 1
AND posted_at <= ?
AND is_permanent = 0
)
LIMIT ${CLEAN_PER_LOOP}
`,
[deletedAtThreshold, postedAtThreshold, botPostedAtThreshold],
);
if (rows.length > 0) {
await repo.delete({
id: In(rows.map(r => r.id)),
});
}
} while (rows.length === CLEAN_PER_LOOP);
setTimeout(cleanup, CLEANUP_INTERVAL);
}
if (!isAPI()) { if (!isAPI()) {
const CLEANUP_INTERVAL = 5 * MINUTES;
async function cleanup() {
await cleanupMessages();
setTimeout(cleanup, CLEANUP_INTERVAL);
}
// Start first cleanup 30 seconds after startup // Start first cleanup 30 seconds after startup
setTimeout(cleanup, 30 * SECONDS); setTimeout(cleanup, 30 * SECONDS);
} }

48
backend/src/data/UsernameHistory.ts
View file

@ -1,9 +1,24 @@
import { getRepository, Repository } from "typeorm"; import { getRepository, In, Repository } from "typeorm";
import { UsernameHistoryEntry } from "./entities/UsernameHistoryEntry"; import { UsernameHistoryEntry } from "./entities/UsernameHistoryEntry";
import { sorter } from "../utils"; import { MINUTES, SECONDS, sorter } from "../utils";
import { BaseRepository } from "./BaseRepository"; import { BaseRepository } from "./BaseRepository";
import { connection } from "./db";
import { isAPI } from "../globals";
import { cleanupUsernames } from "./cleanup/usernames";
export const MAX_USERNAME_ENTRIES_PER_USER = 10; if (!isAPI()) {
const CLEANUP_INTERVAL = 5 * MINUTES;
async function cleanup() {
await cleanupUsernames();
setTimeout(cleanup, CLEANUP_INTERVAL);
}
// Start first cleanup 30 seconds after startup
setTimeout(cleanup, 1 * SECONDS);
}
export const MAX_USERNAME_ENTRIES_PER_USER = 5;
export class UsernameHistory extends BaseRepository { export class UsernameHistory extends BaseRepository {
private usernameHistory: Repository<UsernameHistoryEntry>; private usernameHistory: Repository<UsernameHistoryEntry>;
@ -43,23 +58,18 @@ export class UsernameHistory extends BaseRepository {
}); });
// Cleanup (leave only the last MAX_USERNAME_ENTRIES_PER_USER entries) // Cleanup (leave only the last MAX_USERNAME_ENTRIES_PER_USER entries)
const lastEntries = await this.getByUserId(userId); const toDelete = await this.usernameHistory
if (lastEntries.length > MAX_USERNAME_ENTRIES_PER_USER) { .createQueryBuilder()
const earliestEntry = lastEntries .where("user_id = :userId", { userId })
.sort(sorter("timestamp", "DESC")) .orderBy("id", "DESC")
.slice(0, 10) .skip(MAX_USERNAME_ENTRIES_PER_USER)
.reduce((earliest, entry) => { .take(99_999)
if (earliest == null) return entry; .getMany();
if (entry.id < earliest.id) return entry;
return earliest;
}, null);
this.usernameHistory if (toDelete.length > 0) {
.createQueryBuilder() await this.usernameHistory.delete({
.andWhere("user_id = :userId", { userId }) id: In(toDelete.map(v => v.id)),
.andWhere("id < :id", { id: earliestEntry.id }) });
.delete()
.execute();
} }
} }
} }

68
backend/src/data/cleanup/messages.ts Normal file
View file

@ -0,0 +1,68 @@
import { DAYS, DBDateFormat, MINUTES } from "../../utils";
import { getRepository, In } from "typeorm";
import { SavedMessage } from "../entities/SavedMessage";
import moment from "moment-timezone";
import { connection } from "../db";
/**
* How long message edits, deletions, etc. will include the original message content.
* This is very heavy storage-wise, so keeping it as low as possible is ideal.
*/
const RETENTION_PERIOD = 1 * DAYS;
const BOT_MESSAGE_RETENTION_PERIOD = 30 * MINUTES;
const DELETED_MESSAGE_RETENTION_PERIOD = 5 * MINUTES;
const CLEAN_PER_LOOP = 250;
export async function cleanupMessages(): Promise<number> {
let cleaned = 0;
const messagesRepository = getRepository(SavedMessage);
const deletedAtThreshold = moment()
.subtract(DELETED_MESSAGE_RETENTION_PERIOD, "ms")
.format(DBDateFormat);
const postedAtThreshold = moment()
.subtract(RETENTION_PERIOD, "ms")
.format(DBDateFormat);
const botPostedAtThreshold = moment()
.subtract(BOT_MESSAGE_RETENTION_PERIOD, "ms")
.format(DBDateFormat);
// SELECT + DELETE messages in batches
// This is to avoid deadlocks that happened frequently when deleting with the same criteria as the select below
// when a message was being inserted at the same time
let rows;
do {
rows = await connection.query(
`
SELECT id
FROM messages
WHERE (
deleted_at IS NOT NULL
AND deleted_at <= ?
)
OR (
posted_at <= ?
AND is_permanent = 0
)
OR (
is_bot = 1
AND posted_at <= ?
AND is_permanent = 0
)
LIMIT ${CLEAN_PER_LOOP}
`,
[deletedAtThreshold, postedAtThreshold, botPostedAtThreshold],
);
if (rows.length > 0) {
await messagesRepository.delete({
id: In(rows.map(r => r.id)),
});
}
cleaned += rows.length;
} while (rows.length === CLEAN_PER_LOOP);
return cleaned;
}

41
backend/src/data/cleanup/nicknames.ts Normal file
View file

@ -0,0 +1,41 @@
import { getRepository, In } from "typeorm";
import moment from "moment-timezone";
import { NicknameHistoryEntry } from "../entities/NicknameHistoryEntry";
import { DAYS, DBDateFormat } from "../../utils";
import { connection } from "../db";
export const NICKNAME_RETENTION_PERIOD = 30 * DAYS;
const CLEAN_PER_LOOP = 500;
export async function cleanupNicknames(): Promise<number> {
let cleaned = 0;
const nicknameHistoryRepository = getRepository(NicknameHistoryEntry);
const dateThreshold = moment()
.subtract(NICKNAME_RETENTION_PERIOD, "ms")
.format(DBDateFormat);
// Clean old nicknames (NICKNAME_RETENTION_PERIOD)
let rows;
do {
rows = await connection.query(
`
SELECT id
FROM nickname_history
WHERE timestamp < ?
LIMIT ${CLEAN_PER_LOOP}
`,
[dateThreshold],
);
if (rows.length > 0) {
await nicknameHistoryRepository.delete({
id: In(rows.map(r => r.id)),
});
}
cleaned += rows.length;
} while (rows.length === CLEAN_PER_LOOP);
return cleaned;
}

45
backend/src/data/cleanup/usernames.ts Normal file
View file

@ -0,0 +1,45 @@
import { getRepository, In } from "typeorm";
import moment from "moment-timezone";
import { UsernameHistoryEntry } from "../entities/UsernameHistoryEntry";
import { DAYS, DBDateFormat } from "../../utils";
import { connection } from "../db";
export const USERNAME_RETENTION_PERIOD = 30 * DAYS;
const CLEAN_PER_LOOP = 500;
export async function cleanupUsernames(): Promise<number> {
let cleaned = 0;
const usernameHistoryRepository = getRepository(UsernameHistoryEntry);
const dateThreshold = moment()
.subtract(USERNAME_RETENTION_PERIOD, "ms")
.format(DBDateFormat);
// Clean old usernames (USERNAME_RETENTION_PERIOD)
let rows;
do {
rows = await connection.query(
`
SELECT id
FROM username_history
WHERE timestamp < ?
LIMIT ${CLEAN_PER_LOOP}
`,
[dateThreshold],
);
if (rows.length > 0) {
console.log(
"ids",
rows.map(r => r.id),
);
await usernameHistoryRepository.delete({
id: In(rows.map(r => r.id)),
});
}
cleaned += rows.length;
} while (rows.length === CLEAN_PER_LOOP);
return cleaned;
}

30
backend/src/plugins/NameHistory.ts
View file

@ -1,10 +1,13 @@
import { decorators as d, IPluginOptions } from "knub"; import { decorators as d, IPluginOptions } from "knub";
import { GuildNicknameHistory, MAX_NICKNAME_ENTRIES_PER_USER } from "../data/GuildNicknameHistory"; import { GuildNicknameHistory, MAX_NICKNAME_ENTRIES_PER_USER } from "../data/GuildNicknameHistory";
import { Member, Message } from "eris"; import { Member, Message } from "eris";
import { createChunkedMessage, disableCodeBlocks } from "../utils"; import { createChunkedMessage, DAYS, disableCodeBlocks } from "../utils";
import { ZeppelinPlugin } from "./ZeppelinPlugin"; import { ZeppelinPlugin } from "./ZeppelinPlugin";
import { MAX_USERNAME_ENTRIES_PER_USER, UsernameHistory } from "../data/UsernameHistory"; import { MAX_USERNAME_ENTRIES_PER_USER, UsernameHistory } from "../data/UsernameHistory";
import * as t from "io-ts"; import * as t from "io-ts";
import { NICKNAME_RETENTION_PERIOD } from "../data/cleanup/nicknames";
import moment from "moment-timezone";
import { USERNAME_RETENTION_PERIOD } from "../data/cleanup/usernames";
const ConfigSchema = t.type({ const ConfigSchema = t.type({
can_view: t.boolean, can_view: t.boolean,
@ -59,23 +62,38 @@ export class NameHistoryPlugin extends ZeppelinPlugin<TConfigSchema> {
const user = this.bot.users.get(args.userId); const user = this.bot.users.get(args.userId);
const currentUsername = user ? `${user.username}#${user.discriminator}` : args.userId; const currentUsername = user ? `${user.username}#${user.discriminator}` : args.userId;
const nicknameDays = Math.round(NICKNAME_RETENTION_PERIOD / DAYS);
const usernameDays = Math.round(USERNAME_RETENTION_PERIOD / DAYS);
let message = `Name history for **${currentUsername}**:`; let message = `Name history for **${currentUsername}**:`;
if (nicknameRows.length) { if (nicknameRows.length) {
message += `\n\n__Last ${MAX_NICKNAME_ENTRIES_PER_USER} nicknames:__\n${nicknameRows.join("\n")}`; message += `\n\n__Last ${MAX_NICKNAME_ENTRIES_PER_USER} nicknames within ${nicknameDays} days:__\n${nicknameRows.join(
"\n",
)}`;
} }
if (usernameRows.length) { if (usernameRows.length) {
message += `\n\n__Last ${MAX_USERNAME_ENTRIES_PER_USER} usernames:__\n${usernameRows.join("\n")}`; message += `\n\n__Last ${MAX_USERNAME_ENTRIES_PER_USER} usernames within ${usernameDays} days:__\n${usernameRows.join(
"\n",
)}`;
} }
createChunkedMessage(msg.channel, message); createChunkedMessage(msg.channel, message);
} }
@d.event("guildMemberUpdate") async updateNickname(member: Member) {
async onGuildMemberUpdate(_, member: Member) {
const latestEntry = await this.nicknameHistory.getLastEntry(member.id); const latestEntry = await this.nicknameHistory.getLastEntry(member.id);
if (!latestEntry || latestEntry.nickname !== member.nick) { if (!latestEntry || latestEntry.nickname !== member.nick) {
// tslint:disable-line
await this.nicknameHistory.addEntry(member.id, member.nick); await this.nicknameHistory.addEntry(member.id, member.nick);
} }
} }
@d.event("messageCreate")
async onMessage(msg: Message) {
this.updateNickname(msg.member);
}
@d.event("voiceChannelJoin")
async onVoiceChannelJoin(member: Member) {
this.updateNickname(member);
}
} }

14
backend/src/plugins/UsernameSaver.ts
View file

@ -1,6 +1,6 @@
import { decorators as d, GlobalPlugin } from "knub"; import { decorators as d, GlobalPlugin } from "knub";
import { UsernameHistory } from "../data/UsernameHistory"; import { UsernameHistory } from "../data/UsernameHistory";
import { Member, User } from "eris"; import { Member, Message, User } from "eris";
import { GlobalZeppelinPlugin } from "./GlobalZeppelinPlugin"; import { GlobalZeppelinPlugin } from "./GlobalZeppelinPlugin";
export class UsernameSaver extends GlobalZeppelinPlugin { export class UsernameSaver extends GlobalZeppelinPlugin {
@ -21,13 +21,15 @@ export class UsernameSaver extends GlobalZeppelinPlugin {
} }
} }
@d.event("userUpdate", null, false) @d.event("messageCreate")
async onUserUpdate(user: User) { async onMessage(msg: Message) {
this.updateUsername(user); if (msg.author.bot) return;
this.updateUsername(msg.author);
} }
@d.event("guildMemberAdd", null, false) @d.event("voiceChannelJoin")
async onGuildMemberAdd(_, member: Member) { async onVoiceChannelJoin(member: Member) {
if (member.user.bot) return;
this.updateUsername(member.user); this.updateUsername(member.user);
} }
} }