From c03e7240b41c486b2fa433ba4d7aa36714828364 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Tue, 13 Apr 2021 23:35:19 +0300 Subject: [PATCH 1/3] Attempt to fix occasional deadlock in counter decays --- backend/src/data/GuildCounters.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/src/data/GuildCounters.ts b/backend/src/data/GuildCounters.ts index 234c481f..4cbe1681 100644 --- a/backend/src/data/GuildCounters.ts +++ b/backend/src/data/GuildCounters.ts @@ -186,14 +186,16 @@ export class GuildCounters extends BaseGuildRepository { ? `GREATEST(value - ${decayAmountToApply}, 0)` : `LEAST(value + ${Math.abs(decayAmountToApply)}, ${MAX_COUNTER_VALUE})`; - await this.counterValues.update( - { - counter_id: id, - }, - { + // Using an UPDATE with ORDER BY in an attempt to avoid deadlocks from simultaneous decays + // Also see https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks-handling.html + await this.counterValues + .createQueryBuilder("CounterValue") + .where("counter_id = :id", { id }) + .orderBy("id") + .update({ value: () => rawUpdate, - }, - ); + }) + .execute(); await this.counters.update( { From a558af1038704ae54aac2cb83f54741cb0171a26 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Tue, 13 Apr 2021 23:50:39 +0300 Subject: [PATCH 2/3] Fix crash from passing an invalid regex source to TRegex validation function --- backend/src/validatorUtils.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/src/validatorUtils.ts b/backend/src/validatorUtils.ts index 1db917c0..5292ca96 100644 --- a/backend/src/validatorUtils.ts +++ b/backend/src/validatorUtils.ts @@ -27,7 +27,15 @@ export const TRegex = new t.Type( (s): s is RegExp => s instanceof RegExp, (from, to) => either.chain(t.string.validate(from, to), s => { - return t.success(inputPatternToRegExp(s)); + try { + return t.success(inputPatternToRegExp(s)); + } catch (err) { + if (err instanceof InvalidRegexError) { + return t.failure(s, [], err.message); + } + + throw err; + } }), s => `/${s.source}/${s.flags}`, ); From c26ab2977fb658232d4790b02678fff5157e0a9b Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Wed, 14 Apr 2021 00:19:39 +0300 Subject: [PATCH 3/3] Attempt another fix for counter decay deadlocks --- backend/src/data/GuildCounters.ts | 87 ++++++++++++++++--------------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/backend/src/data/GuildCounters.ts b/backend/src/data/GuildCounters.ts index 4cbe1681..d3c2389a 100644 --- a/backend/src/data/GuildCounters.ts +++ b/backend/src/data/GuildCounters.ts @@ -12,12 +12,15 @@ import { CounterTriggerState } from "./entities/CounterTriggerState"; import moment from "moment-timezone"; import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils"; import { connection } from "./db"; +import { Queue } from "../Queue"; const DELETE_UNUSED_COUNTERS_AFTER = 1 * DAYS; const DELETE_UNUSED_COUNTER_TRIGGERS_AFTER = 1 * DAYS; const MAX_COUNTER_VALUE = 2147483647; // 2^31-1, for MySQL INT +const decayQueue = new Queue(); + async function deleteCountersMarkedToBeDeleted(): Promise { await getRepository(Counter) .createQueryBuilder() @@ -158,53 +161,55 @@ export class GuildCounters extends BaseGuildRepository { ); } - async decay(id: number, decayPeriodMs: number, decayAmount: number) { - const counter = (await this.counters.findOne({ - where: { - id, - }, - }))!; + decay(id: number, decayPeriodMs: number, decayAmount: number) { + return decayQueue.add(async () => { + const counter = (await this.counters.findOne({ + where: { + id, + }, + }))!; - const diffFromLastDecayMs = moment.utc().diff(moment.utc(counter.last_decay_at!), "ms"); - if (diffFromLastDecayMs < decayPeriodMs) { - return; - } + const diffFromLastDecayMs = moment.utc().diff(moment.utc(counter.last_decay_at!), "ms"); + if (diffFromLastDecayMs < decayPeriodMs) { + return; + } - const decayAmountToApply = Math.round((diffFromLastDecayMs / decayPeriodMs) * decayAmount); - if (decayAmountToApply === 0) { - return; - } + const decayAmountToApply = Math.round((diffFromLastDecayMs / decayPeriodMs) * decayAmount); + if (decayAmountToApply === 0) { + return; + } - // Calculate new last_decay_at based on the rounded decay amount we applied. This makes it so that over time, the decayed amount will stay accurate, even if we round some here. - const newLastDecayDate = moment - .utc(counter.last_decay_at) - .add((decayAmountToApply / decayAmount) * decayPeriodMs, "ms") - .format(DBDateFormat); + // Calculate new last_decay_at based on the rounded decay amount we applied. This makes it so that over time, the decayed amount will stay accurate, even if we round some here. + const newLastDecayDate = moment + .utc(counter.last_decay_at) + .add((decayAmountToApply / decayAmount) * decayPeriodMs, "ms") + .format(DBDateFormat); - const rawUpdate = - decayAmountToApply >= 0 - ? `GREATEST(value - ${decayAmountToApply}, 0)` - : `LEAST(value + ${Math.abs(decayAmountToApply)}, ${MAX_COUNTER_VALUE})`; + const rawUpdate = + decayAmountToApply >= 0 + ? `GREATEST(value - ${decayAmountToApply}, 0)` + : `LEAST(value + ${Math.abs(decayAmountToApply)}, ${MAX_COUNTER_VALUE})`; - // Using an UPDATE with ORDER BY in an attempt to avoid deadlocks from simultaneous decays - // Also see https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks-handling.html - await this.counterValues - .createQueryBuilder("CounterValue") - .where("counter_id = :id", { id }) - .orderBy("id") - .update({ - value: () => rawUpdate, - }) - .execute(); + // Using an UPDATE with ORDER BY in an attempt to avoid deadlocks from simultaneous decays + // Also see https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks-handling.html + await this.counterValues + .createQueryBuilder("CounterValue") + .where("counter_id = :id", { id }) + .orderBy("id") + .update({ + value: () => rawUpdate, + }) + .execute(); - await this.counters.update( - { - id, - }, - { - last_decay_at: newLastDecayDate, - }, - ); + await this.counters.update( + { + id, + }, + { + last_decay_at: newLastDecayDate, + }, + ); + }); } async markUnusedTriggersToBeDeleted(triggerIdsToKeep: number[]) {