From 3773d659cc1d773920b7b66498b3f75165bbd192 Mon Sep 17 00:00:00 2001
From: Dragory <2606411+Dragory@users.noreply.github.com>
Date: Sun, 26 Jun 2022 14:34:54 +0300
Subject: [PATCH] Consolidate .env files. More work on dev containers.
---
.env.example | 25 +++++++---
backend/src/api/auth.ts | 27 +++--------
backend/src/api/index.ts | 4 +-
backend/src/api/loadEnv.ts | 4 --
backend/src/api/start.ts | 5 +-
backend/src/data/Phisherman.ts | 3 +-
backend/src/env.ts | 44 ++++++++++++++++++
backend/src/index.ts | 10 +---
backend/src/loadEnv.ts | 4 --
backend/src/plugins/Automod/actions/clean.ts | 17 +------
backend/src/staff.ts | 4 +-
backend/src/utils/crypt.ts | 13 ++----
dashboard/webpack.config.js | 2 +-
docker/development/devenv/Dockerfile | 17 +++++--
docker/development/docker-compose.yml | 48 ++++++++++++--------
docker/development/nginx/Dockerfile | 10 ++--
docker/development/nginx/default.conf | 6 ++-
17 files changed, 137 insertions(+), 106 deletions(-)
delete mode 100644 backend/src/api/loadEnv.ts
create mode 100644 backend/src/env.ts
delete mode 100644 backend/src/loadEnv.ts
diff --git a/.env.example b/.env.example
index 917a8d65..8d8052b9 100644
--- a/.env.example
+++ b/.env.example
@@ -1,4 +1,4 @@
-ENCRYPTION_KEY=32_character_encryption_key
+KEY=32_character_encryption_key
CLIENT_ID=
CLIENT_SECRET=
@@ -7,26 +7,37 @@ BOT_TOKEN=
OAUTH_CALLBACK_URL=
DASHBOARD_DOMAIN=
API_DOMAIN=
-PORT=443
+API_PORT=3000
+
+#
+# DOCKER (DEVELOPMENT)
+#
+
+DOCKER_WEB_PORT=443
# The MySQL database running in the container is exposed to the host on this port,
# allowing access with database tools such as DBeaver
-MYSQL_PORT=3001
+DOCKER_MYSQL_PORT=3001
# Password for the Zeppelin database user
-MYSQL_PASSWORD=
+DOCKER_MYSQL_PASSWORD=
# Password for the MySQL root user
-MYSQL_ROOT_PASSWORD=
+DOCKER_MYSQL_ROOT_PASSWORD=
# The development environment container has an SSH server that you can connect to.
# This is the port that server is exposed to the host on.
-DEVELOPMENT_SSH_PORT=3002
+DOCKER_DEV_SSH_PORT=3002
+DOCKER_DEV_SSH_PASSWORD=password
# Only required if relevant feature is used
#PHISHERMAN_API_KEY=
+#
+# PRODUCTION
+#
+
# In production, the newest code is pulled from a repository
# Specify that repository URL here
-PRODUCTION_REPOSITORY=https://github.com/ZeppelinBot/Zeppelin.git
+#PRODUCTION_REPOSITORY=https://github.com/ZeppelinBot/Zeppelin.git
# You only need to set these if you're running an external database.
# In a standard setup, the database is run in a docker container.
diff --git a/backend/src/api/auth.ts b/backend/src/api/auth.ts
index c1d19b09..09029bda 100644
--- a/backend/src/api/auth.ts
+++ b/backend/src/api/auth.ts
@@ -9,6 +9,7 @@ import { ApiPermissionAssignments } from "../data/ApiPermissionAssignments";
import { ApiUserInfo } from "../data/ApiUserInfo";
import { ApiUserInfoData } from "../data/entities/ApiUserInfo";
import { ok } from "./responses";
+import { env } from "../env";
interface IPassportApiUser {
apiKey: string;
@@ -54,22 +55,6 @@ function simpleDiscordAPIRequest(bearerToken, path): Promise<any> {
export function initAuth(app: express.Express) {
app.use(passport.initialize());
- if (!process.env.CLIENT_ID) {
- throw new Error("Auth: CLIENT ID missing");
- }
-
- if (!process.env.CLIENT_SECRET) {
- throw new Error("Auth: CLIENT SECRET missing");
- }
-
- if (!process.env.OAUTH_CALLBACK_URL) {
- throw new Error("Auth: OAUTH CALLBACK URL missing");
- }
-
- if (!process.env.DASHBOARD_URL) {
- throw new Error("DASHBOARD_URL missing!");
- }
-
passport.serializeUser((user, done) => done(null, user));
passport.deserializeUser((user, done) => done(null, user));
@@ -101,9 +86,9 @@ export function initAuth(app: express.Express) {
{
authorizationURL: "https://discord.com/api/oauth2/authorize",
tokenURL: "https://discord.com/api/oauth2/token",
- clientID: process.env.CLIENT_ID,
- clientSecret: process.env.CLIENT_SECRET,
- callbackURL: process.env.OAUTH_CALLBACK_URL,
+ clientID: env.CLIENT_ID,
+ clientSecret: env.CLIENT_SECRET,
+ callbackURL: env.OAUTH_CALLBACK_URL,
scope: ["identify"],
},
async (accessToken, refreshToken, profile, cb) => {
@@ -132,9 +117,9 @@ export function initAuth(app: express.Express) {
passport.authenticate("oauth2", { failureRedirect: "/", session: false }),
(req: Request, res: Response) => {
if (req.user && req.user.apiKey) {
- res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?apiKey=${req.user.apiKey}`);
+ res.redirect(`https://${env.DASHBOARD_DOMAIN}/login-callback/?apiKey=${req.user.apiKey}`);
} else {
- res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?error=noAccess`);
+ res.redirect(`https://${env.DASHBOARD_DOMAIN}/login-callback/?error=noAccess`);
}
},
);
diff --git a/backend/src/api/index.ts b/backend/src/api/index.ts
index cd2ae878..e8ed699a 100644
--- a/backend/src/api/index.ts
+++ b/backend/src/api/index.ts
@@ -1,8 +1,8 @@
import { connect } from "../data/db";
import { setIsAPI } from "../globals";
-import "./loadEnv";
+import { apiEnv } from "./loadApiEnv";
-if (!process.env.KEY) {
+if (!apiEnv.KEY) {
// tslint:disable-next-line:no-console
console.error("Project root .env with KEY is required!");
process.exit(1);
diff --git a/backend/src/api/loadEnv.ts b/backend/src/api/loadEnv.ts
deleted file mode 100644
index 0bbc5063..00000000
--- a/backend/src/api/loadEnv.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-import path from "path";
-
-require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") });
-require("dotenv").config({ path: path.resolve(process.cwd(), "api.env") });
diff --git a/backend/src/api/start.ts b/backend/src/api/start.ts
index 8089dac0..27d982ab 100644
--- a/backend/src/api/start.ts
+++ b/backend/src/api/start.ts
@@ -8,12 +8,13 @@ import { initGuildsAPI } from "./guilds/index";
import { clientError, error, notFound } from "./responses";
import { startBackgroundTasks } from "./tasks";
import multer from "multer";
+import { env } from "../env";
const app = express();
app.use(
cors({
- origin: process.env.DASHBOARD_URL,
+ origin: `https://${env.DASHBOARD_DOMAIN}`,
}),
);
app.use(
@@ -48,7 +49,7 @@ app.use((req, res, next) => {
return notFound(res);
});
-const port = (process.env.PORT && parseInt(process.env.PORT, 10)) || 3000;
+const port = env.API_PORT;
app.listen(port, "0.0.0.0", () => console.log(`API server listening on port ${port}`)); // tslint:disable-line
startBackgroundTasks();
diff --git a/backend/src/data/Phisherman.ts b/backend/src/data/Phisherman.ts
index 9ab1ef00..2f7b90fc 100644
--- a/backend/src/data/Phisherman.ts
+++ b/backend/src/data/Phisherman.ts
@@ -6,9 +6,10 @@ import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
import moment from "moment-timezone";
import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
import crypto from "crypto";
+import { env } from "../env";
const API_URL = "https://api.phisherman.gg";
-const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
+const MASTER_API_KEY = env.PHISHERMAN_API_KEY;
let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
diff --git a/backend/src/env.ts b/backend/src/env.ts
new file mode 100644
index 00000000..4698d9b3
--- /dev/null
+++ b/backend/src/env.ts
@@ -0,0 +1,44 @@
+import path from "path";
+import fs from "fs";
+import dotenv from "dotenv";
+import { rootDir } from "./paths";
+import { z } from "zod";
+
+const envType = z.object({
+ KEY: z.string().length(32),
+
+ CLIENT_ID: z.string(),
+ CLIENT_SECRET: z.string(),
+ BOT_TOKEN: z.string(),
+
+ OAUTH_CALLBACK_URL: z.string().url(),
+ DASHBOARD_DOMAIN: z.string(),
+ API_DOMAIN: z.string(),
+
+ STAFF: z.preprocess((v) => String(v).split(","), z.array(z.string())).optional(),
+
+ PHISHERMAN_API_KEY: z.string().optional(),
+
+ API_PORT: z.number().min(1).max(65535),
+
+ DOCKER_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development
+
+ DB_HOST: z.string().optional().default("mysql"),
+ DB_PORT: z.number().optional().default(3306),
+ DB_USER: z.string().optional().default("zeppelin"),
+ DB_PASSWORD: z.string().optional(), // Default is set to DOCKER_MYSQL_PASSWORD further below
+ DB_DATABASE: z.string().optional().default("zeppelin"),
+});
+
+let toValidate = {};
+const envPath = path.join(rootDir, "../.env");
+if (fs.existsSync(envPath)) {
+ const buf = fs.readFileSync(envPath);
+ toValidate = dotenv.parse(buf);
+}
+
+export const env = envType.parse(toValidate);
+
+if (env.DOCKER_MYSQL_PASSWORD && !env.DB_PASSWORD) {
+ env.DB_PASSWORD = env.DOCKER_MYSQL_PASSWORD;
+}
diff --git a/backend/src/index.ts b/backend/src/index.ts
index f5438991..31b2c0a8 100644
--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -10,7 +10,6 @@ import { connect } from "./data/db";
import { GuildLogs } from "./data/GuildLogs";
import { LogType } from "./data/LogType";
import { DiscordJSError } from "./DiscordJSError";
-import "./loadEnv";
import { logger } from "./logger";
import { baseGuildPlugins, globalPlugins, guildPlugins } from "./plugins/availablePlugins";
import { RecoverablePluginError } from "./RecoverablePluginError";
@@ -37,12 +36,7 @@ import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./dat
import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
import { consumeQueryStats } from "./data/queryLogger";
import { EventEmitter } from "events";
-
-if (!process.env.KEY) {
- // tslint:disable-next-line:no-console
- console.error("Project root .env with KEY is required!");
- process.exit(1);
-}
+import { env } from "./env";
// Error handling
let recentPluginErrors = 0;
@@ -413,5 +407,5 @@ connect().then(async () => {
bot.initialize();
logger.info("Bot Initialized");
logger.info("Logging in...");
- await client.login(process.env.TOKEN);
+ await client.login(env.BOT_TOKEN);
});
diff --git a/backend/src/loadEnv.ts b/backend/src/loadEnv.ts
deleted file mode 100644
index d0991965..00000000
--- a/backend/src/loadEnv.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-import path from "path";
-
-require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") });
-require("dotenv").config({ path: path.resolve(process.cwd(), "bot.env") });
diff --git a/backend/src/plugins/Automod/actions/clean.ts b/backend/src/plugins/Automod/actions/clean.ts
index 56452d79..27ee1cb6 100644
--- a/backend/src/plugins/Automod/actions/clean.ts
+++ b/backend/src/plugins/Automod/actions/clean.ts
@@ -4,8 +4,6 @@ import { LogType } from "../../../data/LogType";
import { noop } from "../../../utils";
import { automodAction } from "../helpers";
-const cleanDebugServer = process.env.TEMP_CLEAN_DEBUG_SERVER;
-
export const CleanAction = automodAction({
configType: t.boolean,
defaultConfig: false,
@@ -29,26 +27,13 @@ export const CleanAction = automodAction({
}
}
- if (pluginData.guild.id === cleanDebugServer) {
- const toDeleteFormatted = Array.from(messageIdsToDeleteByChannelId.entries())
- .map(([channelId, messageIds]) => `- ${channelId}: ${messageIds.join(", ")}`)
- .join("\n");
- // tslint:disable-next-line:no-console
- console.log(`[DEBUG] Cleaning messages (${ruleName}):\n${toDeleteFormatted}`);
- }
-
for (const [channelId, messageIds] of messageIdsToDeleteByChannelId.entries()) {
for (const id of messageIds) {
pluginData.state.logs.ignoreLog(LogType.MESSAGE_DELETE, id);
}
const channel = pluginData.guild.channels.cache.get(channelId as Snowflake) as TextChannel;
- await channel.bulkDelete(messageIds as Snowflake[]).catch((err) => {
- if (pluginData.guild.id === cleanDebugServer) {
- // tslint:disable-next-line:no-console
- console.error(`[DEBUG] Failed to bulk delete messages (${ruleName}): ${err}`);
- }
- });
+ await channel.bulkDelete(messageIds as Snowflake[]).catch(noop);
}
},
});
diff --git a/backend/src/staff.ts b/backend/src/staff.ts
index 7c14f0b6..6950ee33 100644
--- a/backend/src/staff.ts
+++ b/backend/src/staff.ts
@@ -1,6 +1,8 @@
+import { env } from "./env";
+
/**
* Zeppelin staff have full access to the dashboard
*/
export function isStaff(userId: string) {
- return (process.env.STAFF ?? "").split(",").includes(userId);
+ return (env.STAFF ?? []).includes(userId);
}
diff --git a/backend/src/utils/crypt.ts b/backend/src/utils/crypt.ts
index bba4c09a..d6208f1c 100644
--- a/backend/src/utils/crypt.ts
+++ b/backend/src/utils/crypt.ts
@@ -1,21 +1,14 @@
import { spawn, Worker, Pool } from "threads";
-import "../loadEnv";
import type { CryptFns } from "./cryptWorker";
import { MINUTES } from "../utils";
+import { env } from "../env";
-if (!process.env.KEY) {
- // tslint:disable-next-line:no-console
- console.error("Environment value KEY required for encryption");
- process.exit(1);
-}
-
-const KEY = process.env.KEY;
const pool = Pool(() => spawn(new Worker("./cryptWorker"), { timeout: 10 * MINUTES }), 8);
export async function encrypt(data: string) {
- return pool.queue((w) => w.encrypt(data, KEY));
+ return pool.queue((w) => w.encrypt(data, env.KEY));
}
export async function decrypt(data: string) {
- return pool.queue((w) => w.decrypt(data, KEY));
+ return pool.queue((w) => w.decrypt(data, env.KEY));
}
diff --git a/dashboard/webpack.config.js b/dashboard/webpack.config.js
index e4b78765..7fb9f221 100644
--- a/dashboard/webpack.config.js
+++ b/dashboard/webpack.config.js
@@ -1,4 +1,4 @@
-require("dotenv").config();
+require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") });
const path = require("path");
const VueLoaderPlugin = require("vue-loader/lib/plugin");
diff --git a/docker/development/devenv/Dockerfile b/docker/development/devenv/Dockerfile
index 95e3e128..ac278942 100644
--- a/docker/development/devenv/Dockerfile
+++ b/docker/development/devenv/Dockerfile
@@ -1,18 +1,27 @@
FROM ubuntu:20.04
ARG DOCKER_UID
+ARG DOCKER_DEV_SSH_PASSWORD
ENV DEBIAN_FRONTEND=noninteractive
ENV TZ=UTC
+# Set up some core packages
+RUN apt-get update
+RUN apt-get install -y sudo git curl
+
# Set up SSH access
-RUN apt-get update && apt-get install -y openssh-server sudo git
+RUN apt-get install -y openssh-server iptables
RUN mkdir /var/run/sshd
RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u "${DOCKER_UID}" ubuntu
-RUN echo 'ubuntu:password' | chpasswd
+RUN echo "ubuntu:${DOCKER_DEV_SSH_PASSWORD}" | chpasswd
-# Install Node.js 16
+# Set up proper permissions for volumes
+RUN mkdir -p /home/ubuntu/zeppelin /home/ubuntu/.vscode-remote /home/ubuntu/.vscode-server /home/ubuntu/.cache/JetBrains
+RUN chown -R ubuntu /home/ubuntu
+
+# Install Node.js 16 and packages needed to build native packages
RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
-RUN apt-get install -y nodejs
+RUN apt-get install -y nodejs gcc g++ make python3
CMD /usr/sbin/sshd -D -e
diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml
index c5721914..351cf21d 100644
--- a/docker/development/docker-compose.yml
+++ b/docker/development/docker-compose.yml
@@ -1,28 +1,33 @@
version: '3'
+volumes:
+ mysql-data: {}
+ vscode-remote: {}
+ vscode-server: {}
+ jetbrains-data: {}
services:
-# nginx:
-# user: "${UID:?Missing UID}:${GID:?Missing GID}"
-# build:
-# context: ./nginx
-# args:
-# API_DOMAIN: ${API_DOMAIN:?Missing API_DOMAIN}
-# API_PORT: ${API_PORT:?Missing API_PORT}
-# DASHBOARD_DOMAIN: ${DASHBOARD_DOMAIN:?Missing DASHBOARD_DOMAIN}
-# DASHBOARD_PORT: ${DASHBOARD_PORT:?Missing DASHBOARD_PORT}
-# ports:
-# - ${PORT:?Missing PORT}:443
-# volumes:
-# - ./:/zeppelin
-#
+ nginx:
+ build:
+ context: ./nginx
+ args:
+ API_DOMAIN: ${API_DOMAIN:?Missing API_DOMAIN}
+ API_PORT: ${API_PORT:?Missing API_PORT}
+ DASHBOARD_DOMAIN: ${DASHBOARD_DOMAIN:?Missing DASHBOARD_DOMAIN}
+ ports:
+ - ${DOCKER_WEB_PORT:?Missing DOCKER_WEB_PORT}:443
+ volumes:
+ - ../../:/zeppelin
+
mysql:
image: mysql:8.0
environment:
- MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD?:Missing MYSQL_ROOT_PASSWORD}
+ MYSQL_ROOT_PASSWORD: ${DOCKER_MYSQL_ROOT_PASSWORD?:Missing DOCKER_MYSQL_ROOT_PASSWORD}
MYSQL_DATABASE: zeppelin
MYSQL_USER: zeppelin
- MYSQL_PASSWORD: ${MYSQL_PASSWORD?:Missing MYSQL_PASSWORD}
+ MYSQL_PASSWORD: ${DOCKER_MYSQL_PASSWORD?:Missing DOCKER_MYSQL_PASSWORD}
ports:
- - ${MYSQL_PORT:?Missing MYSQL_PORT}:3306
+ - ${DOCKER_MYSQL_PORT:?Missing DOCKER_MYSQL_PORT}:3306
+ volumes:
+ - mysql-data:/var/lib/mysql
#
# backend:
# image: node:16
@@ -50,7 +55,12 @@ services:
args:
DOCKER_UID: ${DOCKER_UID:?Missing DOCKER_UID}
DOCKER_GID: ${DOCKER_GID:?Missing DOCKER_GID}
+ DOCKER_DEV_SSH_PASSWORD: ${DOCKER_DEV_SSH_PASSWORD:?Missing DOCKER_DEV_SSH_PASSWORD}
ports:
- - "${DEVELOPMENT_SSH_PORT:?Missing DEVELOPMENT_SSH_PORT}:22"
+ - "${DOCKER_DEV_SSH_PORT:?Missing DOCKER_DEV_SSH_PORT}:22"
volumes:
- - ../../:/zeppelin
+ - ../../:/home/ubuntu/zeppelin
+ - ~/.ssh:/home/ubuntu/.ssh
+ - vscode-remote:/home/ubuntu/.vscode-remote
+ - vscode-server:/home/ubuntu/.vscode-server
+ - jetbrains-data:/home/ubuntu/.cache/JetBrains
diff --git a/docker/development/nginx/Dockerfile b/docker/development/nginx/Dockerfile
index 7058f8b0..2f63ba2f 100644
--- a/docker/development/nginx/Dockerfile
+++ b/docker/development/nginx/Dockerfile
@@ -2,11 +2,13 @@ FROM nginx
ARG API_DOMAIN
ARG DASHBOARD_DOMAIN
+ARG API_PORT
RUN apt-get update && apt-get install -y openssl
-RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/api-cert.key -out /etc/ssl/certs/api-cert.pem -days 365 -subj '/CN=*.${API_DOMAIN}' -nodes
-RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/dashboard-cert.key -out /etc/ssl/certs/dashboard-cert.pem -days 365 -subj '/CN=*.${DASHBOARD_DOMAIN}' -nodes
+RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/api-cert.key -out /etc/ssl/certs/api-cert.pem -days 3650 -subj '/CN=*.${API_DOMAIN}' -nodes
+RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/dashboard-cert.key -out /etc/ssl/certs/dashboard-cert.pem -days 3650 -subj '/CN=*.${DASHBOARD_DOMAIN}' -nodes
COPY ./default.conf /etc/nginx/conf.d/default.conf
-RUN sed -ir "s/_API_DOMAIN_/$(echo ${API_DOMAIN} | sed -ir 's///g')/g"
-RUN sed -ir "s/_DASHBOARD_DOMAIN_/$(echo ${DASHBOARD_DOMAIN} | sed 's/\./\\\\./g')/g"
+RUN sed -ir "s/_API_DOMAIN_/$(echo ${API_DOMAIN} | sed 's/\./\\./g')/g" /etc/nginx/conf.d/default.conf
+RUN sed -ir "s/_DASHBOARD_DOMAIN_/$(echo ${DASHBOARD_DOMAIN} | sed 's/\./\\./g')/g" /etc/nginx/conf.d/default.conf
+RUN sed -ir "s/_API_PORT_/${API_PORT}/g" /etc/nginx/conf.d/default.conf
diff --git a/docker/development/nginx/default.conf b/docker/development/nginx/default.conf
index 6aea2aeb..126df16b 100644
--- a/docker/development/nginx/default.conf
+++ b/docker/development/nginx/default.conf
@@ -4,7 +4,9 @@ server {
server_name _API_DOMAIN_;
location / {
- proxy_pass backend:3000;
+ # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable
+ set $backend_upstream devenv;
+ proxy_pass http://$backend_upstream:_API_PORT_;
client_max_body_size 200M;
}
@@ -23,7 +25,7 @@ server {
server {
listen 443 ssl http2;
listen [::]:443 ssl http2;
- server_name dashboard.dev.zeppelin.gg;
+ server_name _DASHBOARD_DOMAIN_;
root /zeppelin/dashboard/dist;