3
0
Fork 0
mirror of https://github.com/ZeppelinBot/Zeppelin.git synced 2025-05-10 04:25:01 +00:00

Merge branch '220601_docker_wip'

This commit is contained in:
Dragory 2022-08-07 14:02:44 +03:00
commit 218c31231e
No known key found for this signature in database
GPG key ID: 5F387BA66DF8AAC1
61 changed files with 738 additions and 312 deletions

View file

@ -9,6 +9,7 @@ import { ApiPermissionAssignments } from "../data/ApiPermissionAssignments";
import { ApiUserInfo } from "../data/ApiUserInfo";
import { ApiUserInfoData } from "../data/entities/ApiUserInfo";
import { ok } from "./responses";
import { env } from "../env";
interface IPassportApiUser {
apiKey: string;
@ -54,22 +55,6 @@ function simpleDiscordAPIRequest(bearerToken, path): Promise<any> {
export function initAuth(app: express.Express) {
app.use(passport.initialize());
if (!process.env.CLIENT_ID) {
throw new Error("Auth: CLIENT ID missing");
}
if (!process.env.CLIENT_SECRET) {
throw new Error("Auth: CLIENT SECRET missing");
}
if (!process.env.OAUTH_CALLBACK_URL) {
throw new Error("Auth: OAUTH CALLBACK URL missing");
}
if (!process.env.DASHBOARD_URL) {
throw new Error("DASHBOARD_URL missing!");
}
passport.serializeUser((user, done) => done(null, user));
passport.deserializeUser((user, done) => done(null, user));
@ -101,9 +86,9 @@ export function initAuth(app: express.Express) {
{
authorizationURL: "https://discord.com/api/oauth2/authorize",
tokenURL: "https://discord.com/api/oauth2/token",
clientID: process.env.CLIENT_ID,
clientSecret: process.env.CLIENT_SECRET,
callbackURL: process.env.OAUTH_CALLBACK_URL,
clientID: env.CLIENT_ID,
clientSecret: env.CLIENT_SECRET,
callbackURL: `${env.API_URL}/auth/oauth-callback`,
scope: ["identify"],
},
async (accessToken, refreshToken, profile, cb) => {
@ -132,9 +117,9 @@ export function initAuth(app: express.Express) {
passport.authenticate("oauth2", { failureRedirect: "/", session: false }),
(req: Request, res: Response) => {
if (req.user && req.user.apiKey) {
res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?apiKey=${req.user.apiKey}`);
res.redirect(`${env.DASHBOARD_URL}/login-callback/?apiKey=${req.user.apiKey}`);
} else {
res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?error=noAccess`);
res.redirect(`${env.DASHBOARD_URL}/login-callback/?error=noAccess`);
}
},
);

View file

@ -1,8 +1,8 @@
import { connect } from "../data/db";
import { setIsAPI } from "../globals";
import "./loadEnv";
import { env } from "../env";
if (!process.env.KEY) {
if (!env.KEY) {
// tslint:disable-next-line:no-console
console.error("Project root .env with KEY is required!");
process.exit(1);

View file

@ -1,4 +0,0 @@
import path from "path";
require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") });
require("dotenv").config({ path: path.resolve(process.cwd(), "api.env") });

View file

@ -8,12 +8,13 @@ import { initGuildsAPI } from "./guilds/index";
import { clientError, error, notFound } from "./responses";
import { startBackgroundTasks } from "./tasks";
import multer from "multer";
import { env } from "../env";
const app = express();
app.use(
cors({
origin: process.env.DASHBOARD_URL,
origin: env.DASHBOARD_URL,
}),
);
app.use(
@ -48,7 +49,7 @@ app.use((req, res, next) => {
return notFound(res);
});
const port = (process.env.PORT && parseInt(process.env.PORT, 10)) || 3000;
const port = env.API_PORT;
app.listen(port, "0.0.0.0", () => console.log(`API server listening on port ${port}`)); // tslint:disable-line
startBackgroundTasks();

View file

@ -4,6 +4,7 @@ import { BaseRepository } from "./BaseRepository";
import { AllowedGuild } from "./entities/AllowedGuild";
import moment from "moment-timezone";
import { DBDateFormat } from "../utils";
import { env } from "../env";
export class AllowedGuilds extends BaseRepository {
private allowedGuilds: Repository<AllowedGuild>;

View file

@ -6,9 +6,10 @@ import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
import moment from "moment-timezone";
import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
import crypto from "crypto";
import { env } from "../env";
const API_URL = "https://api.phisherman.gg";
const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
const MASTER_API_KEY = env.PHISHERMAN_API_KEY;
let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();

View file

@ -9,6 +9,9 @@ const CLEAN_PER_LOOP = 50;
export async function cleanupConfigs() {
const configRepository = getRepository(Config);
// FIXME: The query below doesn't work on MySQL 8.0. Pending an update.
return;
let cleaned = 0;
let rows;

View file

@ -1,7 +1,11 @@
import { Connection, createConnection } from "typeorm";
import { SimpleError } from "../SimpleError";
import connectionOptions from "../../ormconfig";
import { QueryLogger } from "./queryLogger";
import path from "path";
import { backendDir } from "../paths";
const ormconfigPath = path.join(backendDir, "ormconfig.js");
const connectionOptions = require(ormconfigPath);
let connectionPromise: Promise<Connection>;

70
backend/src/env.ts Normal file
View file

@ -0,0 +1,70 @@
import path from "path";
import fs from "fs";
import dotenv from "dotenv";
import { rootDir } from "./paths";
import { z } from "zod";
const envType = z.object({
KEY: z.string().length(32),
CLIENT_ID: z.string().min(16),
CLIENT_SECRET: z.string().length(32),
BOT_TOKEN: z.string().min(50),
DASHBOARD_URL: z.string().url(),
API_URL: z.string().url(),
API_PORT: z.preprocess((v) => Number(v), z.number().min(1).max(65535)).default(3000),
STAFF: z
.preprocess(
(v) =>
String(v)
.split(",")
.map((s) => s.trim())
.filter((s) => s !== ""),
z.array(z.string()),
)
.optional(),
DEFAULT_ALLOWED_SERVERS: z
.preprocess(
(v) =>
String(v)
.split(",")
.map((s) => s.trim())
.filter((s) => s !== ""),
z.array(z.string()),
)
.optional(),
PHISHERMAN_API_KEY: z.string().optional(),
DOCKER_DEV_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development
DOCKER_PROD_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in production
DB_HOST: z.string().optional().default("mysql"),
DB_PORT: z
.preprocess((v) => Number(v), z.number())
.optional()
.default(3306),
DB_USER: z.string().optional().default("zeppelin"),
DB_PASSWORD: z.string().optional(), // Default is set to DOCKER_MYSQL_PASSWORD further below
DB_DATABASE: z.string().optional().default("zeppelin"),
});
let toValidate = {};
const envPath = path.join(rootDir, ".env");
if (fs.existsSync(envPath)) {
const buf = fs.readFileSync(envPath);
toValidate = dotenv.parse(buf);
}
export const env = envType.parse(toValidate);
if (!env.DB_PASSWORD) {
if (process.env.NODE_ENV === "production" && env.DOCKER_PROD_MYSQL_PASSWORD) {
env.DB_PASSWORD = env.DOCKER_PROD_MYSQL_PASSWORD;
} else if (env.DOCKER_DEV_MYSQL_PASSWORD) {
env.DB_PASSWORD = env.DOCKER_DEV_MYSQL_PASSWORD;
}
}

View file

@ -10,7 +10,6 @@ import { connect } from "./data/db";
import { GuildLogs } from "./data/GuildLogs";
import { LogType } from "./data/LogType";
import { DiscordJSError } from "./DiscordJSError";
import "./loadEnv";
import { logger } from "./logger";
import { baseGuildPlugins, globalPlugins, guildPlugins } from "./plugins/availablePlugins";
import { RecoverablePluginError } from "./RecoverablePluginError";
@ -37,12 +36,7 @@ import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./dat
import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
import { consumeQueryStats } from "./data/queryLogger";
import { EventEmitter } from "events";
if (!process.env.KEY) {
// tslint:disable-next-line:no-console
console.error("Project root .env with KEY is required!");
process.exit(1);
}
import { env } from "./env";
// Error handling
let recentPluginErrors = 0;
@ -413,5 +407,5 @@ connect().then(async () => {
bot.initialize();
logger.info("Bot Initialized");
logger.info("Logging in...");
await client.login(process.env.TOKEN);
await client.login(env.BOT_TOKEN);
});

View file

@ -1,4 +0,0 @@
import path from "path";
require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") });
require("dotenv").config({ path: path.resolve(process.cwd(), "bot.env") });

View file

@ -14,6 +14,7 @@ import { TZeppelinKnub } from "./types";
import { deepKeyIntersect, errorMessage, successMessage, tDeepPartial, tNullable } from "./utils";
import { Tail } from "./utils/typeUtils";
import { decodeAndValidateStrict, StrictValidationError, validate } from "./validatorUtils";
import { isStaff } from "./staff";
const { getMemberLevel } = helpers;
@ -242,8 +243,8 @@ export function isOwner(pluginData: AnyPluginData<any>, userId: string) {
return owners.includes(userId);
}
export const isOwnerPreFilter = (_, context: CommandContext<any>) => {
return isOwner(context.pluginData, context.message.author.id);
export const isStaffPreFilter = (_, context: CommandContext<any>) => {
return isStaff(context.message.author.id);
};
type AnyFn = (...args: any[]) => any;

View file

@ -4,8 +4,6 @@ import { LogType } from "../../../data/LogType";
import { noop } from "../../../utils";
import { automodAction } from "../helpers";
const cleanDebugServer = process.env.TEMP_CLEAN_DEBUG_SERVER;
export const CleanAction = automodAction({
configType: t.boolean,
defaultConfig: false,
@ -29,26 +27,13 @@ export const CleanAction = automodAction({
}
}
if (pluginData.guild.id === cleanDebugServer) {
const toDeleteFormatted = Array.from(messageIdsToDeleteByChannelId.entries())
.map(([channelId, messageIds]) => `- ${channelId}: ${messageIds.join(", ")}`)
.join("\n");
// tslint:disable-next-line:no-console
console.log(`[DEBUG] Cleaning messages (${ruleName}):\n${toDeleteFormatted}`);
}
for (const [channelId, messageIds] of messageIdsToDeleteByChannelId.entries()) {
for (const id of messageIds) {
pluginData.state.logs.ignoreLog(LogType.MESSAGE_DELETE, id);
}
const channel = pluginData.guild.channels.cache.get(channelId as Snowflake) as TextChannel;
await channel.bulkDelete(messageIds as Snowflake[]).catch((err) => {
if (pluginData.guild.id === cleanDebugServer) {
// tslint:disable-next-line:no-console
console.error(`[DEBUG] Failed to bulk delete messages (${ruleName}): ${err}`);
}
});
await channel.bulkDelete(messageIds as Snowflake[]).catch(noop);
}
},
});

View file

@ -1,14 +1,14 @@
import { ApiPermissions } from "@shared/apiPermissions";
import { TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { botControlCmd } from "../types";
export const AddDashboardUserCmd = botControlCmd({
trigger: ["add_dashboard_user"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -1,7 +1,7 @@
import { ApiPermissions } from "@shared/apiPermissions";
import { TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { DBDateFormat, isGuildInvite, isSnowflake, resolveInvite } from "../../../utils";
import { botControlCmd } from "../types";
import moment from "moment-timezone";

View file

@ -1,7 +1,7 @@
import { ApiPermissions } from "@shared/apiPermissions";
import { TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { DBDateFormat, isSnowflake } from "../../../utils";
import { botControlCmd } from "../types";
import moment from "moment-timezone";
@ -10,7 +10,7 @@ export const AllowServerCmd = botControlCmd({
trigger: ["allow_server", "allowserver", "add_server", "addserver"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -1,6 +1,6 @@
import { Guild, GuildChannel, TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { GuildInvite, isGuildInvite, resolveInvite, verboseUserMention } from "../../../utils";
import { botControlCmd } from "../types";
import { isEligible } from "../functions/isEligible";
@ -9,7 +9,7 @@ export const ChannelToServerCmd = botControlCmd({
trigger: ["channel_to_server", "channel2server"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -1,6 +1,6 @@
import { Snowflake, TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { noop } from "../../../utils";
import { botControlCmd } from "../types";
@ -8,7 +8,7 @@ export const DisallowServerCmd = botControlCmd({
trigger: ["disallow_server", "disallowserver", "remove_server", "removeserver"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -1,13 +1,13 @@
import { Snowflake, TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { botControlCmd } from "../types";
export const LeaveServerCmd = botControlCmd({
trigger: ["leave_server", "leave_guild"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -2,7 +2,7 @@ import { TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { AllowedGuild } from "../../../data/entities/AllowedGuild";
import { ApiPermissionAssignment } from "../../../data/entities/ApiPermissionAssignment";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { resolveUser } from "../../../utils";
import { botControlCmd } from "../types";

View file

@ -1,6 +1,6 @@
import { TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { resolveUser } from "../../../utils";
import { botControlCmd } from "../types";

View file

@ -1,5 +1,5 @@
import { TextChannel } from "discord.js";
import { isOwnerPreFilter } from "../../../pluginUtils";
import { isStaffPreFilter } from "../../../pluginUtils";
import { getActiveReload, setActiveReload } from "../activeReload";
import { botControlCmd } from "../types";
@ -7,7 +7,7 @@ export const ReloadGlobalPluginsCmd = botControlCmd({
trigger: "bot_reload_global_plugins",
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
async run({ pluginData, message }) {

View file

@ -1,13 +1,13 @@
import { Snowflake, TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { botControlCmd } from "../types";
export const ReloadServerCmd = botControlCmd({
trigger: ["reload_server", "reload_guild"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -1,13 +1,13 @@
import { TextChannel } from "discord.js";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
import { botControlCmd } from "../types";
export const RemoveDashboardUserCmd = botControlCmd({
trigger: ["remove_dashboard_user"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -1,7 +1,7 @@
import { TextChannel } from "discord.js";
import escapeStringRegexp from "escape-string-regexp";
import { commandTypeHelpers as ct } from "../../../commandTypes";
import { isOwnerPreFilter } from "../../../pluginUtils";
import { isStaffPreFilter } from "../../../pluginUtils";
import { createChunkedMessage, getUser, sorter } from "../../../utils";
import { botControlCmd } from "../types";
@ -9,7 +9,7 @@ export const ServersCmd = botControlCmd({
trigger: ["servers", "guilds"],
permission: null,
config: {
preFilters: [isOwnerPreFilter],
preFilters: [isStaffPreFilter],
},
signature: {

View file

@ -3,6 +3,8 @@ import * as t from "io-ts";
import { BasePluginType, GlobalPluginData, typedGlobalEventListener } from "knub";
import { AllowedGuilds } from "../../data/AllowedGuilds";
import { zeppelinGlobalPlugin } from "../ZeppelinPluginBlueprint";
import { env } from "../../env";
import { Configs } from "../../data/Configs";
interface GuildAccessMonitorPluginType extends BasePluginType {
config: {};
@ -15,7 +17,7 @@ async function checkGuild(pluginData: GlobalPluginData<GuildAccessMonitorPluginT
if (!(await pluginData.state.allowedGuilds.isAllowed(guild.id))) {
// tslint:disable-next-line:no-console
console.log(`Non-allowed server ${guild.name} (${guild.id}), leaving`);
guild.leave();
// guild.leave();
}
}
@ -35,8 +37,19 @@ export const GuildAccessMonitorPlugin = zeppelinGlobalPlugin<GuildAccessMonitorP
}),
],
beforeLoad(pluginData) {
async beforeLoad(pluginData) {
pluginData.state.allowedGuilds = new AllowedGuilds();
const defaultAllowedServers = env.DEFAULT_ALLOWED_SERVERS || [];
const configs = new Configs();
for (const serverId of defaultAllowedServers) {
if (!(await pluginData.state.allowedGuilds.isAllowed(serverId))) {
// tslint:disable-next-line:no-console
console.log(`Adding allowed-by-default server ${serverId} to the allowed servers`);
await pluginData.state.allowedGuilds.add(serverId);
await configs.saveNewRevision(`guild-${serverId}`, "plugins: {}", 0);
}
}
},
afterLoad(pluginData) {

View file

@ -1,6 +1,8 @@
import { env } from "./env";
/**
* Zeppelin staff have full access to the dashboard
*/
export function isStaff(userId: string) {
return (process.env.STAFF ?? "").split(",").includes(userId);
return (env.STAFF ?? []).includes(userId);
}

View file

@ -1,21 +1,14 @@
import { spawn, Worker, Pool } from "threads";
import "../loadEnv";
import type { CryptFns } from "./cryptWorker";
import { MINUTES } from "../utils";
import { env } from "../env";
if (!process.env.KEY) {
// tslint:disable-next-line:no-console
console.error("Environment value KEY required for encryption");
process.exit(1);
}
const KEY = process.env.KEY;
const pool = Pool(() => spawn(new Worker("./cryptWorker"), { timeout: 10 * MINUTES }), 8);
export async function encrypt(data: string) {
return pool.queue((w) => w.encrypt(data, KEY));
return pool.queue((w) => w.encrypt(data, env.KEY));
}
export async function decrypt(data: string) {
return pool.queue((w) => w.decrypt(data, KEY));
return pool.queue((w) => w.decrypt(data, env.KEY));
}