From 2a959f354c263b6aea985176e55d5503f9dd37de Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Wed, 1 Jun 2022 19:11:44 +0300 Subject: [PATCH 01/29] Initial work on Docker support --- .env.example | 37 +++++++++++++++++- backend/api.env.example | 10 ----- backend/bot.env.example | 7 ---- docker-compose-dev.sh | 3 ++ docker/development/devenv/Dockerfile | 18 +++++++++ docker/development/docker-compose.yml | 56 +++++++++++++++++++++++++++ docker/development/nginx/Dockerfile | 12 ++++++ docker/development/nginx/default.conf | 44 +++++++++++++++++++++ 8 files changed, 169 insertions(+), 18 deletions(-) delete mode 100644 backend/api.env.example delete mode 100644 backend/bot.env.example create mode 100755 docker-compose-dev.sh create mode 100644 docker/development/devenv/Dockerfile create mode 100644 docker/development/docker-compose.yml create mode 100644 docker/development/nginx/Dockerfile create mode 100644 docker/development/nginx/default.conf diff --git a/.env.example b/.env.example index ab4597a4..917a8d65 100644 --- a/.env.example +++ b/.env.example @@ -1 +1,36 @@ -KEY=32_character_encryption_key +ENCRYPTION_KEY=32_character_encryption_key + +CLIENT_ID= +CLIENT_SECRET= +BOT_TOKEN= + +OAUTH_CALLBACK_URL= +DASHBOARD_DOMAIN= +API_DOMAIN= +PORT=443 + +# The MySQL database running in the container is exposed to the host on this port, +# allowing access with database tools such as DBeaver +MYSQL_PORT=3001 +# Password for the Zeppelin database user +MYSQL_PASSWORD= +# Password for the MySQL root user +MYSQL_ROOT_PASSWORD= + +# The development environment container has an SSH server that you can connect to. +# This is the port that server is exposed to the host on. +DEVELOPMENT_SSH_PORT=3002 + +# Only required if relevant feature is used +#PHISHERMAN_API_KEY= + +# In production, the newest code is pulled from a repository +# Specify that repository URL here +PRODUCTION_REPOSITORY=https://github.com/ZeppelinBot/Zeppelin.git + +# You only need to set these if you're running an external database. +# In a standard setup, the database is run in a docker container. +#DB_HOST= +#DB_USER= +#DB_PASSWORD= +#DB_DATABASE= diff --git a/backend/api.env.example b/backend/api.env.example deleted file mode 100644 index 87afe866..00000000 --- a/backend/api.env.example +++ /dev/null @@ -1,10 +0,0 @@ -PORT= -CLIENT_ID= -CLIENT_SECRET= -OAUTH_CALLBACK_URL= -DASHBOARD_URL= -DB_HOST= -DB_USER= -DB_PASSWORD= -DB_DATABASE= -STAFF= diff --git a/backend/bot.env.example b/backend/bot.env.example deleted file mode 100644 index b701f4ed..00000000 --- a/backend/bot.env.example +++ /dev/null @@ -1,7 +0,0 @@ -TOKEN= -DB_HOST= -DB_USER= -DB_PASSWORD= -DB_DATABASE= -PROFILING=false -PHISHERMAN_API_KEY= diff --git a/docker-compose-dev.sh b/docker-compose-dev.sh new file mode 100755 index 00000000..14af4e94 --- /dev/null +++ b/docker-compose-dev.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +DOCKER_UID="$(id -u)" DOCKER_GID="$(id -g)" docker-compose --env-file ./.env -f ./docker/development/docker-compose.yml "$@" diff --git a/docker/development/devenv/Dockerfile b/docker/development/devenv/Dockerfile new file mode 100644 index 00000000..95e3e128 --- /dev/null +++ b/docker/development/devenv/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:20.04 + +ARG DOCKER_UID + +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=UTC + +# Set up SSH access +RUN apt-get update && apt-get install -y openssh-server sudo git +RUN mkdir /var/run/sshd +RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u "${DOCKER_UID}" ubuntu +RUN echo 'ubuntu:password' | chpasswd + +# Install Node.js 16 +RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - +RUN apt-get install -y nodejs + +CMD /usr/sbin/sshd -D -e diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml new file mode 100644 index 00000000..c5721914 --- /dev/null +++ b/docker/development/docker-compose.yml @@ -0,0 +1,56 @@ +version: '3' +services: +# nginx: +# user: "${UID:?Missing UID}:${GID:?Missing GID}" +# build: +# context: ./nginx +# args: +# API_DOMAIN: ${API_DOMAIN:?Missing API_DOMAIN} +# API_PORT: ${API_PORT:?Missing API_PORT} +# DASHBOARD_DOMAIN: ${DASHBOARD_DOMAIN:?Missing DASHBOARD_DOMAIN} +# DASHBOARD_PORT: ${DASHBOARD_PORT:?Missing DASHBOARD_PORT} +# ports: +# - ${PORT:?Missing PORT}:443 +# volumes: +# - ./:/zeppelin +# + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD?:Missing MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: zeppelin + MYSQL_USER: zeppelin + MYSQL_PASSWORD: ${MYSQL_PASSWORD?:Missing MYSQL_PASSWORD} + ports: + - ${MYSQL_PORT:?Missing MYSQL_PORT}:3306 +# +# backend: +# image: node:16 +# user: "${UID:?Missing UID}:${GID:?Missing GID}" +# working_dir: /zeppelin/backend +# restart: always +# depends_on: +# - mysql +# volumes: +# - ./:/zeppelin +# command: sh -c "npm run migrate-dev && npm run watch" +# +# dashboard: +# image: node:16 +# user: "${UID:?Missing UID}:${GID:?Missing GID}" +# working_dir: /zeppelin/dashboard +# restart: always +# volumes: +# - ./:/zeppelin +# command: sh -c "npm run watch-build" + + devenv: + build: + context: ./devenv + args: + DOCKER_UID: ${DOCKER_UID:?Missing DOCKER_UID} + DOCKER_GID: ${DOCKER_GID:?Missing DOCKER_GID} + ports: + - "${DEVELOPMENT_SSH_PORT:?Missing DEVELOPMENT_SSH_PORT}:22" + volumes: + - ../../:/zeppelin diff --git a/docker/development/nginx/Dockerfile b/docker/development/nginx/Dockerfile new file mode 100644 index 00000000..7058f8b0 --- /dev/null +++ b/docker/development/nginx/Dockerfile @@ -0,0 +1,12 @@ +FROM nginx + +ARG API_DOMAIN +ARG DASHBOARD_DOMAIN + +RUN apt-get update && apt-get install -y openssl +RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/api-cert.key -out /etc/ssl/certs/api-cert.pem -days 365 -subj '/CN=*.${API_DOMAIN}' -nodes +RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/dashboard-cert.key -out /etc/ssl/certs/dashboard-cert.pem -days 365 -subj '/CN=*.${DASHBOARD_DOMAIN}' -nodes + +COPY ./default.conf /etc/nginx/conf.d/default.conf +RUN sed -ir "s/_API_DOMAIN_/$(echo ${API_DOMAIN} | sed -ir 's///g')/g" +RUN sed -ir "s/_DASHBOARD_DOMAIN_/$(echo ${DASHBOARD_DOMAIN} | sed 's/\./\\\\./g')/g" diff --git a/docker/development/nginx/default.conf b/docker/development/nginx/default.conf new file mode 100644 index 00000000..6aea2aeb --- /dev/null +++ b/docker/development/nginx/default.conf @@ -0,0 +1,44 @@ +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name _API_DOMAIN_; + + location / { + proxy_pass backend:3000; + + client_max_body_size 200M; + } + + ssl_certificate /etc/ssl/certs/api-cert.pem; + ssl_certificate_key /etc/ssl/private/api-cert.key; + + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; + ssl_session_tickets off; + + ssl_protocols TLSv1.3; + ssl_prefer_server_ciphers off; +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name dashboard.dev.zeppelin.gg; + + root /zeppelin/dashboard/dist; + + location / { + index index.html; + try_files $uri $uri/ /index.html; + } + + ssl_certificate /etc/ssl/certs/dashboard-cert.pem; + ssl_certificate_key /etc/ssl/private/dashboard-cert.key; + + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; + ssl_session_tickets off; + + ssl_protocols TLSv1.3; + ssl_prefer_server_ciphers off; +} From 3773d659cc1d773920b7b66498b3f75165bbd192 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:34:54 +0300 Subject: [PATCH 02/29] Consolidate .env files. More work on dev containers. --- .env.example | 25 +++++++--- backend/src/api/auth.ts | 27 +++-------- backend/src/api/index.ts | 4 +- backend/src/api/loadEnv.ts | 4 -- backend/src/api/start.ts | 5 +- backend/src/data/Phisherman.ts | 3 +- backend/src/env.ts | 44 ++++++++++++++++++ backend/src/index.ts | 10 +--- backend/src/loadEnv.ts | 4 -- backend/src/plugins/Automod/actions/clean.ts | 17 +------ backend/src/staff.ts | 4 +- backend/src/utils/crypt.ts | 13 ++---- dashboard/webpack.config.js | 2 +- docker/development/devenv/Dockerfile | 17 +++++-- docker/development/docker-compose.yml | 48 ++++++++++++-------- docker/development/nginx/Dockerfile | 10 ++-- docker/development/nginx/default.conf | 6 ++- 17 files changed, 137 insertions(+), 106 deletions(-) delete mode 100644 backend/src/api/loadEnv.ts create mode 100644 backend/src/env.ts delete mode 100644 backend/src/loadEnv.ts diff --git a/.env.example b/.env.example index 917a8d65..8d8052b9 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ -ENCRYPTION_KEY=32_character_encryption_key +KEY=32_character_encryption_key CLIENT_ID= CLIENT_SECRET= @@ -7,26 +7,37 @@ BOT_TOKEN= OAUTH_CALLBACK_URL= DASHBOARD_DOMAIN= API_DOMAIN= -PORT=443 +API_PORT=3000 + +# +# DOCKER (DEVELOPMENT) +# + +DOCKER_WEB_PORT=443 # The MySQL database running in the container is exposed to the host on this port, # allowing access with database tools such as DBeaver -MYSQL_PORT=3001 +DOCKER_MYSQL_PORT=3001 # Password for the Zeppelin database user -MYSQL_PASSWORD= +DOCKER_MYSQL_PASSWORD= # Password for the MySQL root user -MYSQL_ROOT_PASSWORD= +DOCKER_MYSQL_ROOT_PASSWORD= # The development environment container has an SSH server that you can connect to. # This is the port that server is exposed to the host on. -DEVELOPMENT_SSH_PORT=3002 +DOCKER_DEV_SSH_PORT=3002 +DOCKER_DEV_SSH_PASSWORD=password # Only required if relevant feature is used #PHISHERMAN_API_KEY= +# +# PRODUCTION +# + # In production, the newest code is pulled from a repository # Specify that repository URL here -PRODUCTION_REPOSITORY=https://github.com/ZeppelinBot/Zeppelin.git +#PRODUCTION_REPOSITORY=https://github.com/ZeppelinBot/Zeppelin.git # You only need to set these if you're running an external database. # In a standard setup, the database is run in a docker container. diff --git a/backend/src/api/auth.ts b/backend/src/api/auth.ts index c1d19b09..09029bda 100644 --- a/backend/src/api/auth.ts +++ b/backend/src/api/auth.ts @@ -9,6 +9,7 @@ import { ApiPermissionAssignments } from "../data/ApiPermissionAssignments"; import { ApiUserInfo } from "../data/ApiUserInfo"; import { ApiUserInfoData } from "../data/entities/ApiUserInfo"; import { ok } from "./responses"; +import { env } from "../env"; interface IPassportApiUser { apiKey: string; @@ -54,22 +55,6 @@ function simpleDiscordAPIRequest(bearerToken, path): Promise { export function initAuth(app: express.Express) { app.use(passport.initialize()); - if (!process.env.CLIENT_ID) { - throw new Error("Auth: CLIENT ID missing"); - } - - if (!process.env.CLIENT_SECRET) { - throw new Error("Auth: CLIENT SECRET missing"); - } - - if (!process.env.OAUTH_CALLBACK_URL) { - throw new Error("Auth: OAUTH CALLBACK URL missing"); - } - - if (!process.env.DASHBOARD_URL) { - throw new Error("DASHBOARD_URL missing!"); - } - passport.serializeUser((user, done) => done(null, user)); passport.deserializeUser((user, done) => done(null, user)); @@ -101,9 +86,9 @@ export function initAuth(app: express.Express) { { authorizationURL: "https://discord.com/api/oauth2/authorize", tokenURL: "https://discord.com/api/oauth2/token", - clientID: process.env.CLIENT_ID, - clientSecret: process.env.CLIENT_SECRET, - callbackURL: process.env.OAUTH_CALLBACK_URL, + clientID: env.CLIENT_ID, + clientSecret: env.CLIENT_SECRET, + callbackURL: env.OAUTH_CALLBACK_URL, scope: ["identify"], }, async (accessToken, refreshToken, profile, cb) => { @@ -132,9 +117,9 @@ export function initAuth(app: express.Express) { passport.authenticate("oauth2", { failureRedirect: "/", session: false }), (req: Request, res: Response) => { if (req.user && req.user.apiKey) { - res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?apiKey=${req.user.apiKey}`); + res.redirect(`https://${env.DASHBOARD_DOMAIN}/login-callback/?apiKey=${req.user.apiKey}`); } else { - res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?error=noAccess`); + res.redirect(`https://${env.DASHBOARD_DOMAIN}/login-callback/?error=noAccess`); } }, ); diff --git a/backend/src/api/index.ts b/backend/src/api/index.ts index cd2ae878..e8ed699a 100644 --- a/backend/src/api/index.ts +++ b/backend/src/api/index.ts @@ -1,8 +1,8 @@ import { connect } from "../data/db"; import { setIsAPI } from "../globals"; -import "./loadEnv"; +import { apiEnv } from "./loadApiEnv"; -if (!process.env.KEY) { +if (!apiEnv.KEY) { // tslint:disable-next-line:no-console console.error("Project root .env with KEY is required!"); process.exit(1); diff --git a/backend/src/api/loadEnv.ts b/backend/src/api/loadEnv.ts deleted file mode 100644 index 0bbc5063..00000000 --- a/backend/src/api/loadEnv.ts +++ /dev/null @@ -1,4 +0,0 @@ -import path from "path"; - -require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") }); -require("dotenv").config({ path: path.resolve(process.cwd(), "api.env") }); diff --git a/backend/src/api/start.ts b/backend/src/api/start.ts index 8089dac0..27d982ab 100644 --- a/backend/src/api/start.ts +++ b/backend/src/api/start.ts @@ -8,12 +8,13 @@ import { initGuildsAPI } from "./guilds/index"; import { clientError, error, notFound } from "./responses"; import { startBackgroundTasks } from "./tasks"; import multer from "multer"; +import { env } from "../env"; const app = express(); app.use( cors({ - origin: process.env.DASHBOARD_URL, + origin: `https://${env.DASHBOARD_DOMAIN}`, }), ); app.use( @@ -48,7 +49,7 @@ app.use((req, res, next) => { return notFound(res); }); -const port = (process.env.PORT && parseInt(process.env.PORT, 10)) || 3000; +const port = env.API_PORT; app.listen(port, "0.0.0.0", () => console.log(`API server listening on port ${port}`)); // tslint:disable-line startBackgroundTasks(); diff --git a/backend/src/data/Phisherman.ts b/backend/src/data/Phisherman.ts index 9ab1ef00..2f7b90fc 100644 --- a/backend/src/data/Phisherman.ts +++ b/backend/src/data/Phisherman.ts @@ -6,9 +6,10 @@ import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils"; import moment from "moment-timezone"; import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry"; import crypto from "crypto"; +import { env } from "../env"; const API_URL = "https://api.phisherman.gg"; -const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY; +const MASTER_API_KEY = env.PHISHERMAN_API_KEY; let caughtDomainTrackingMap: Map> = new Map(); diff --git a/backend/src/env.ts b/backend/src/env.ts new file mode 100644 index 00000000..4698d9b3 --- /dev/null +++ b/backend/src/env.ts @@ -0,0 +1,44 @@ +import path from "path"; +import fs from "fs"; +import dotenv from "dotenv"; +import { rootDir } from "./paths"; +import { z } from "zod"; + +const envType = z.object({ + KEY: z.string().length(32), + + CLIENT_ID: z.string(), + CLIENT_SECRET: z.string(), + BOT_TOKEN: z.string(), + + OAUTH_CALLBACK_URL: z.string().url(), + DASHBOARD_DOMAIN: z.string(), + API_DOMAIN: z.string(), + + STAFF: z.preprocess((v) => String(v).split(","), z.array(z.string())).optional(), + + PHISHERMAN_API_KEY: z.string().optional(), + + API_PORT: z.number().min(1).max(65535), + + DOCKER_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development + + DB_HOST: z.string().optional().default("mysql"), + DB_PORT: z.number().optional().default(3306), + DB_USER: z.string().optional().default("zeppelin"), + DB_PASSWORD: z.string().optional(), // Default is set to DOCKER_MYSQL_PASSWORD further below + DB_DATABASE: z.string().optional().default("zeppelin"), +}); + +let toValidate = {}; +const envPath = path.join(rootDir, "../.env"); +if (fs.existsSync(envPath)) { + const buf = fs.readFileSync(envPath); + toValidate = dotenv.parse(buf); +} + +export const env = envType.parse(toValidate); + +if (env.DOCKER_MYSQL_PASSWORD && !env.DB_PASSWORD) { + env.DB_PASSWORD = env.DOCKER_MYSQL_PASSWORD; +} diff --git a/backend/src/index.ts b/backend/src/index.ts index f5438991..31b2c0a8 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -10,7 +10,6 @@ import { connect } from "./data/db"; import { GuildLogs } from "./data/GuildLogs"; import { LogType } from "./data/LogType"; import { DiscordJSError } from "./DiscordJSError"; -import "./loadEnv"; import { logger } from "./logger"; import { baseGuildPlugins, globalPlugins, guildPlugins } from "./plugins/availablePlugins"; import { RecoverablePluginError } from "./RecoverablePluginError"; @@ -37,12 +36,7 @@ import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./dat import { hasPhishermanMasterAPIKey } from "./data/Phisherman"; import { consumeQueryStats } from "./data/queryLogger"; import { EventEmitter } from "events"; - -if (!process.env.KEY) { - // tslint:disable-next-line:no-console - console.error("Project root .env with KEY is required!"); - process.exit(1); -} +import { env } from "./env"; // Error handling let recentPluginErrors = 0; @@ -413,5 +407,5 @@ connect().then(async () => { bot.initialize(); logger.info("Bot Initialized"); logger.info("Logging in..."); - await client.login(process.env.TOKEN); + await client.login(env.BOT_TOKEN); }); diff --git a/backend/src/loadEnv.ts b/backend/src/loadEnv.ts deleted file mode 100644 index d0991965..00000000 --- a/backend/src/loadEnv.ts +++ /dev/null @@ -1,4 +0,0 @@ -import path from "path"; - -require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") }); -require("dotenv").config({ path: path.resolve(process.cwd(), "bot.env") }); diff --git a/backend/src/plugins/Automod/actions/clean.ts b/backend/src/plugins/Automod/actions/clean.ts index 56452d79..27ee1cb6 100644 --- a/backend/src/plugins/Automod/actions/clean.ts +++ b/backend/src/plugins/Automod/actions/clean.ts @@ -4,8 +4,6 @@ import { LogType } from "../../../data/LogType"; import { noop } from "../../../utils"; import { automodAction } from "../helpers"; -const cleanDebugServer = process.env.TEMP_CLEAN_DEBUG_SERVER; - export const CleanAction = automodAction({ configType: t.boolean, defaultConfig: false, @@ -29,26 +27,13 @@ export const CleanAction = automodAction({ } } - if (pluginData.guild.id === cleanDebugServer) { - const toDeleteFormatted = Array.from(messageIdsToDeleteByChannelId.entries()) - .map(([channelId, messageIds]) => `- ${channelId}: ${messageIds.join(", ")}`) - .join("\n"); - // tslint:disable-next-line:no-console - console.log(`[DEBUG] Cleaning messages (${ruleName}):\n${toDeleteFormatted}`); - } - for (const [channelId, messageIds] of messageIdsToDeleteByChannelId.entries()) { for (const id of messageIds) { pluginData.state.logs.ignoreLog(LogType.MESSAGE_DELETE, id); } const channel = pluginData.guild.channels.cache.get(channelId as Snowflake) as TextChannel; - await channel.bulkDelete(messageIds as Snowflake[]).catch((err) => { - if (pluginData.guild.id === cleanDebugServer) { - // tslint:disable-next-line:no-console - console.error(`[DEBUG] Failed to bulk delete messages (${ruleName}): ${err}`); - } - }); + await channel.bulkDelete(messageIds as Snowflake[]).catch(noop); } }, }); diff --git a/backend/src/staff.ts b/backend/src/staff.ts index 7c14f0b6..6950ee33 100644 --- a/backend/src/staff.ts +++ b/backend/src/staff.ts @@ -1,6 +1,8 @@ +import { env } from "./env"; + /** * Zeppelin staff have full access to the dashboard */ export function isStaff(userId: string) { - return (process.env.STAFF ?? "").split(",").includes(userId); + return (env.STAFF ?? []).includes(userId); } diff --git a/backend/src/utils/crypt.ts b/backend/src/utils/crypt.ts index bba4c09a..d6208f1c 100644 --- a/backend/src/utils/crypt.ts +++ b/backend/src/utils/crypt.ts @@ -1,21 +1,14 @@ import { spawn, Worker, Pool } from "threads"; -import "../loadEnv"; import type { CryptFns } from "./cryptWorker"; import { MINUTES } from "../utils"; +import { env } from "../env"; -if (!process.env.KEY) { - // tslint:disable-next-line:no-console - console.error("Environment value KEY required for encryption"); - process.exit(1); -} - -const KEY = process.env.KEY; const pool = Pool(() => spawn(new Worker("./cryptWorker"), { timeout: 10 * MINUTES }), 8); export async function encrypt(data: string) { - return pool.queue((w) => w.encrypt(data, KEY)); + return pool.queue((w) => w.encrypt(data, env.KEY)); } export async function decrypt(data: string) { - return pool.queue((w) => w.decrypt(data, KEY)); + return pool.queue((w) => w.decrypt(data, env.KEY)); } diff --git a/dashboard/webpack.config.js b/dashboard/webpack.config.js index e4b78765..7fb9f221 100644 --- a/dashboard/webpack.config.js +++ b/dashboard/webpack.config.js @@ -1,4 +1,4 @@ -require("dotenv").config(); +require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") }); const path = require("path"); const VueLoaderPlugin = require("vue-loader/lib/plugin"); diff --git a/docker/development/devenv/Dockerfile b/docker/development/devenv/Dockerfile index 95e3e128..ac278942 100644 --- a/docker/development/devenv/Dockerfile +++ b/docker/development/devenv/Dockerfile @@ -1,18 +1,27 @@ FROM ubuntu:20.04 ARG DOCKER_UID +ARG DOCKER_DEV_SSH_PASSWORD ENV DEBIAN_FRONTEND=noninteractive ENV TZ=UTC +# Set up some core packages +RUN apt-get update +RUN apt-get install -y sudo git curl + # Set up SSH access -RUN apt-get update && apt-get install -y openssh-server sudo git +RUN apt-get install -y openssh-server iptables RUN mkdir /var/run/sshd RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u "${DOCKER_UID}" ubuntu -RUN echo 'ubuntu:password' | chpasswd +RUN echo "ubuntu:${DOCKER_DEV_SSH_PASSWORD}" | chpasswd -# Install Node.js 16 +# Set up proper permissions for volumes +RUN mkdir -p /home/ubuntu/zeppelin /home/ubuntu/.vscode-remote /home/ubuntu/.vscode-server /home/ubuntu/.cache/JetBrains +RUN chown -R ubuntu /home/ubuntu + +# Install Node.js 16 and packages needed to build native packages RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - -RUN apt-get install -y nodejs +RUN apt-get install -y nodejs gcc g++ make python3 CMD /usr/sbin/sshd -D -e diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index c5721914..351cf21d 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -1,28 +1,33 @@ version: '3' +volumes: + mysql-data: {} + vscode-remote: {} + vscode-server: {} + jetbrains-data: {} services: -# nginx: -# user: "${UID:?Missing UID}:${GID:?Missing GID}" -# build: -# context: ./nginx -# args: -# API_DOMAIN: ${API_DOMAIN:?Missing API_DOMAIN} -# API_PORT: ${API_PORT:?Missing API_PORT} -# DASHBOARD_DOMAIN: ${DASHBOARD_DOMAIN:?Missing DASHBOARD_DOMAIN} -# DASHBOARD_PORT: ${DASHBOARD_PORT:?Missing DASHBOARD_PORT} -# ports: -# - ${PORT:?Missing PORT}:443 -# volumes: -# - ./:/zeppelin -# + nginx: + build: + context: ./nginx + args: + API_DOMAIN: ${API_DOMAIN:?Missing API_DOMAIN} + API_PORT: ${API_PORT:?Missing API_PORT} + DASHBOARD_DOMAIN: ${DASHBOARD_DOMAIN:?Missing DASHBOARD_DOMAIN} + ports: + - ${DOCKER_WEB_PORT:?Missing DOCKER_WEB_PORT}:443 + volumes: + - ../../:/zeppelin + mysql: image: mysql:8.0 environment: - MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD?:Missing MYSQL_ROOT_PASSWORD} + MYSQL_ROOT_PASSWORD: ${DOCKER_MYSQL_ROOT_PASSWORD?:Missing DOCKER_MYSQL_ROOT_PASSWORD} MYSQL_DATABASE: zeppelin MYSQL_USER: zeppelin - MYSQL_PASSWORD: ${MYSQL_PASSWORD?:Missing MYSQL_PASSWORD} + MYSQL_PASSWORD: ${DOCKER_MYSQL_PASSWORD?:Missing DOCKER_MYSQL_PASSWORD} ports: - - ${MYSQL_PORT:?Missing MYSQL_PORT}:3306 + - ${DOCKER_MYSQL_PORT:?Missing DOCKER_MYSQL_PORT}:3306 + volumes: + - mysql-data:/var/lib/mysql # # backend: # image: node:16 @@ -50,7 +55,12 @@ services: args: DOCKER_UID: ${DOCKER_UID:?Missing DOCKER_UID} DOCKER_GID: ${DOCKER_GID:?Missing DOCKER_GID} + DOCKER_DEV_SSH_PASSWORD: ${DOCKER_DEV_SSH_PASSWORD:?Missing DOCKER_DEV_SSH_PASSWORD} ports: - - "${DEVELOPMENT_SSH_PORT:?Missing DEVELOPMENT_SSH_PORT}:22" + - "${DOCKER_DEV_SSH_PORT:?Missing DOCKER_DEV_SSH_PORT}:22" volumes: - - ../../:/zeppelin + - ../../:/home/ubuntu/zeppelin + - ~/.ssh:/home/ubuntu/.ssh + - vscode-remote:/home/ubuntu/.vscode-remote + - vscode-server:/home/ubuntu/.vscode-server + - jetbrains-data:/home/ubuntu/.cache/JetBrains diff --git a/docker/development/nginx/Dockerfile b/docker/development/nginx/Dockerfile index 7058f8b0..2f63ba2f 100644 --- a/docker/development/nginx/Dockerfile +++ b/docker/development/nginx/Dockerfile @@ -2,11 +2,13 @@ FROM nginx ARG API_DOMAIN ARG DASHBOARD_DOMAIN +ARG API_PORT RUN apt-get update && apt-get install -y openssl -RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/api-cert.key -out /etc/ssl/certs/api-cert.pem -days 365 -subj '/CN=*.${API_DOMAIN}' -nodes -RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/dashboard-cert.key -out /etc/ssl/certs/dashboard-cert.pem -days 365 -subj '/CN=*.${DASHBOARD_DOMAIN}' -nodes +RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/api-cert.key -out /etc/ssl/certs/api-cert.pem -days 3650 -subj '/CN=*.${API_DOMAIN}' -nodes +RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/dashboard-cert.key -out /etc/ssl/certs/dashboard-cert.pem -days 3650 -subj '/CN=*.${DASHBOARD_DOMAIN}' -nodes COPY ./default.conf /etc/nginx/conf.d/default.conf -RUN sed -ir "s/_API_DOMAIN_/$(echo ${API_DOMAIN} | sed -ir 's///g')/g" -RUN sed -ir "s/_DASHBOARD_DOMAIN_/$(echo ${DASHBOARD_DOMAIN} | sed 's/\./\\\\./g')/g" +RUN sed -ir "s/_API_DOMAIN_/$(echo ${API_DOMAIN} | sed 's/\./\\./g')/g" /etc/nginx/conf.d/default.conf +RUN sed -ir "s/_DASHBOARD_DOMAIN_/$(echo ${DASHBOARD_DOMAIN} | sed 's/\./\\./g')/g" /etc/nginx/conf.d/default.conf +RUN sed -ir "s/_API_PORT_/${API_PORT}/g" /etc/nginx/conf.d/default.conf diff --git a/docker/development/nginx/default.conf b/docker/development/nginx/default.conf index 6aea2aeb..126df16b 100644 --- a/docker/development/nginx/default.conf +++ b/docker/development/nginx/default.conf @@ -4,7 +4,9 @@ server { server_name _API_DOMAIN_; location / { - proxy_pass backend:3000; + # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable + set $backend_upstream devenv; + proxy_pass http://$backend_upstream:_API_PORT_; client_max_body_size 200M; } @@ -23,7 +25,7 @@ server { server { listen 443 ssl http2; listen [::]:443 ssl http2; - server_name dashboard.dev.zeppelin.gg; + server_name _DASHBOARD_DOMAIN_; root /zeppelin/dashboard/dist; From acbeddf11c7c7fa70194d8e13cfe5c35c2638047 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:37:37 +0300 Subject: [PATCH 03/29] Fix old/broken env import --- backend/src/api/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/index.ts b/backend/src/api/index.ts index e8ed699a..75bcea5e 100644 --- a/backend/src/api/index.ts +++ b/backend/src/api/index.ts @@ -1,8 +1,8 @@ import { connect } from "../data/db"; import { setIsAPI } from "../globals"; -import { apiEnv } from "./loadApiEnv"; +import { env } from "../env"; -if (!apiEnv.KEY) { +if (!env.KEY) { // tslint:disable-next-line:no-console console.error("Project root .env with KEY is required!"); process.exit(1); From 12274a84b27b26d36208f000f07d5c17a3f5b186 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 14:41:31 +0300 Subject: [PATCH 04/29] Fix .env number validation errors --- backend/src/env.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/env.ts b/backend/src/env.ts index 4698d9b3..13cbbdb7 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -19,19 +19,22 @@ const envType = z.object({ PHISHERMAN_API_KEY: z.string().optional(), - API_PORT: z.number().min(1).max(65535), + API_PORT: z.preprocess((v) => Number(v), z.number().min(1).max(65535)), DOCKER_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development DB_HOST: z.string().optional().default("mysql"), - DB_PORT: z.number().optional().default(3306), + DB_PORT: z + .preprocess((v) => Number(v), z.number()) + .optional() + .default(3306), DB_USER: z.string().optional().default("zeppelin"), DB_PASSWORD: z.string().optional(), // Default is set to DOCKER_MYSQL_PASSWORD further below DB_DATABASE: z.string().optional().default("zeppelin"), }); let toValidate = {}; -const envPath = path.join(rootDir, "../.env"); +const envPath = path.join(rootDir, ".env"); if (fs.existsSync(envPath)) { const buf = fs.readFileSync(envPath); toValidate = dotenv.parse(buf); From e760654c544bb54626d404448f0eb0ffa45b0a1a Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:01:54 +0300 Subject: [PATCH 05/29] Use mysql_native_password for dev database zeppelin user --- docker/development/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 351cf21d..10c3caf1 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -28,6 +28,7 @@ services: - ${DOCKER_MYSQL_PORT:?Missing DOCKER_MYSQL_PORT}:3306 volumes: - mysql-data:/var/lib/mysql + command: --authentication-policy=mysql_native_password # # backend: # image: node:16 From 6b44027eb4a0b77a0373f87ededcf6b2527a9dfa Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:02:34 +0300 Subject: [PATCH 06/29] Fix error when compiling ormconfig --- backend/ormconfig.js | 35 ++++++++++------------------------- backend/src/data/db.ts | 6 +++++- backend/src/env.ts | 6 +++--- backend/tsconfig.json | 2 +- 4 files changed, 19 insertions(+), 30 deletions(-) diff --git a/backend/ormconfig.js b/backend/ormconfig.js index c62e60c1..f49cfb58 100644 --- a/backend/ormconfig.js +++ b/backend/ormconfig.js @@ -1,38 +1,23 @@ const fs = require("fs"); const path = require("path"); const pkgUp = require("pkg-up"); - -const closestPackageJson = pkgUp.sync(); -if (!closestPackageJson) { - throw new Error("Could not find project root from ormconfig.js"); -} -const backendRoot = path.dirname(closestPackageJson); - -try { - fs.accessSync(path.resolve(backendRoot, "bot.env")); - require("dotenv").config({ path: path.resolve(backendRoot, "bot.env") }); -} catch { - try { - fs.accessSync(path.resolve(backendRoot, "api.env")); - require("dotenv").config({ path: path.resolve(backendRoot, "api.env") }); - } catch { - throw new Error("bot.env or api.env required"); - } -} +const { backendDir } = require("./dist/backend/src/paths"); +const { env } = require("./dist/backend/src/env"); const moment = require("moment-timezone"); moment.tz.setDefault("UTC"); -const entities = path.relative(process.cwd(), path.resolve(backendRoot, "dist/backend/src/data/entities/*.js")); -const migrations = path.relative(process.cwd(), path.resolve(backendRoot, "dist/backend/src/migrations/*.js")); -const migrationsDir = path.relative(process.cwd(), path.resolve(backendRoot, "src/migrations")); +const entities = path.relative(process.cwd(), path.resolve(backendDir, "dist/backend/src/data/entities/*.js")); +const migrations = path.relative(process.cwd(), path.resolve(backendDir, "dist/backend/src/migrations/*.js")); +const migrationsDir = path.relative(process.cwd(), path.resolve(backendDir, "src/migrations")); module.exports = { type: "mysql", - host: process.env.DB_HOST, - username: process.env.DB_USER, - password: process.env.DB_PASSWORD, - database: process.env.DB_DATABASE, + host: env.DB_HOST, + port: env.DB_PORT, + username: env.DB_USER, + password: env.DB_PASSWORD, + database: env.DB_DATABASE, charset: "utf8mb4", supportBigNumbers: true, bigNumberStrings: true, diff --git a/backend/src/data/db.ts b/backend/src/data/db.ts index b38bbc72..fd42f361 100644 --- a/backend/src/data/db.ts +++ b/backend/src/data/db.ts @@ -1,7 +1,11 @@ import { Connection, createConnection } from "typeorm"; import { SimpleError } from "../SimpleError"; -import connectionOptions from "../../ormconfig"; import { QueryLogger } from "./queryLogger"; +import path from "path"; +import { backendDir } from "../paths"; + +const ormconfigPath = path.join(backendDir, "ormconfig.js"); +const connectionOptions = require(ormconfigPath); let connectionPromise: Promise; diff --git a/backend/src/env.ts b/backend/src/env.ts index 13cbbdb7..b0eb3d9e 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -7,9 +7,9 @@ import { z } from "zod"; const envType = z.object({ KEY: z.string().length(32), - CLIENT_ID: z.string(), - CLIENT_SECRET: z.string(), - BOT_TOKEN: z.string(), + CLIENT_ID: z.string().min(16), + CLIENT_SECRET: z.string().length(32), + BOT_TOKEN: z.string().min(50), OAUTH_CALLBACK_URL: z.string().url(), DASHBOARD_DOMAIN: z.string(), diff --git a/backend/tsconfig.json b/backend/tsconfig.json index 011e0217..cde827e4 100644 --- a/backend/tsconfig.json +++ b/backend/tsconfig.json @@ -24,5 +24,5 @@ "useUnknownInCatchVariables": false, "allowJs": true }, - "include": ["src/**/*.ts", "ormconfig.js"] + "include": ["src/**/*.ts"] } From f463abb3e226e2395c7a2c9fa7ab410950e1dbe7 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:23:17 +0300 Subject: [PATCH 07/29] Temporarily disable config clean-up The current query doesn't work in MySQL 8. --- backend/src/data/cleanup/configs.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/src/data/cleanup/configs.ts b/backend/src/data/cleanup/configs.ts index 1d22ea52..58c6da24 100644 --- a/backend/src/data/cleanup/configs.ts +++ b/backend/src/data/cleanup/configs.ts @@ -9,6 +9,9 @@ const CLEAN_PER_LOOP = 50; export async function cleanupConfigs() { const configRepository = getRepository(Config); + // FIXME: The query below doesn't work on MySQL 8.0. Pending an update. + return; + let cleaned = 0; let rows; From 122f535e34fde91d7c8ade9c826e755c002d9ca9 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:23:53 +0300 Subject: [PATCH 08/29] Fix API vhost in nginx container --- docker/development/nginx/default.conf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/docker/development/nginx/default.conf b/docker/development/nginx/default.conf index 126df16b..3ff41fe0 100644 --- a/docker/development/nginx/default.conf +++ b/docker/development/nginx/default.conf @@ -3,10 +3,15 @@ server { listen [::]:443 ssl http2; server_name _API_DOMAIN_; + location / { # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable - set $backend_upstream devenv; - proxy_pass http://$backend_upstream:_API_PORT_; + set $backend_upstream "http://devenv:_API_PORT_"; + # Using a variable in proxy_pass also requires resolver to be set. + # This is the address of the internal docker compose DNS server. + resolver 127.0.0.11; + + proxy_pass $backend_upstream; client_max_body_size 200M; } From bc250209d84cfaecbbcddf8b2c57a3196d79d448 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:24:24 +0300 Subject: [PATCH 09/29] Fix API_URL/API_DOMAIN usage in dashboard --- dashboard/src/api.ts | 2 +- dashboard/src/auth.ts | 4 ++-- dashboard/src/init-vue.ts | 2 +- dashboard/webpack.config.js | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dashboard/src/api.ts b/dashboard/src/api.ts index f6cfd5e5..6b75d1f8 100644 --- a/dashboard/src/api.ts +++ b/dashboard/src/api.ts @@ -1,5 +1,5 @@ import { RootStore } from "./store"; -const apiUrl = process.env.API_URL; +const apiUrl = `https://${process.env.API_DOMAIN}`; type QueryParamObject = { [key: string]: string | null }; diff --git a/dashboard/src/auth.ts b/dashboard/src/auth.ts index bb5c1b9e..e8c085d7 100644 --- a/dashboard/src/auth.ts +++ b/dashboard/src/auth.ts @@ -11,7 +11,7 @@ const isAuthenticated = async () => { export const authGuard: NavigationGuard = async (to, from, next) => { if (await isAuthenticated()) return next(); - window.location.href = `${process.env.API_URL}/auth/login`; + window.location.href = `https://${process.env.API_DOMAIN}/auth/login`; }; export const loginCallbackGuard: NavigationGuard = async (to, from, next) => { @@ -25,5 +25,5 @@ export const loginCallbackGuard: NavigationGuard = async (to, from, next) => { export const authRedirectGuard: NavigationGuard = async (to, form, next) => { if (await isAuthenticated()) return next("/dashboard"); - window.location.href = `${process.env.API_URL}/auth/login`; + window.location.href = `https://${process.env.API_DOMAIN}/auth/login`; }; diff --git a/dashboard/src/init-vue.ts b/dashboard/src/init-vue.ts index 614b5609..25b30969 100644 --- a/dashboard/src/init-vue.ts +++ b/dashboard/src/init-vue.ts @@ -20,7 +20,7 @@ Vue.mixin({ return { get env() { return Object.freeze({ - API_URL: process.env.API_URL, + API_URL: `https://${process.env.API_DOMAIN}`, }); }, }; diff --git a/dashboard/webpack.config.js b/dashboard/webpack.config.js index 7fb9f221..2ae878ca 100644 --- a/dashboard/webpack.config.js +++ b/dashboard/webpack.config.js @@ -1,5 +1,3 @@ -require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") }); - const path = require("path"); const VueLoaderPlugin = require("vue-loader/lib/plugin"); const HtmlWebpackPlugin = require("html-webpack-plugin"); @@ -154,7 +152,9 @@ let config = { js: ["./src/main.ts"], }, }), - new DotenvPlugin(), + new DotenvPlugin({ + path: path.resolve(process.cwd(), "../.env"), + }), ], resolve: { extensions: [".ts", ".tsx", ".js", ".mjs", ".vue"], From b6550851154301559fab003418c9d45823c145e9 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 15:31:20 +0300 Subject: [PATCH 10/29] Add quick instructions for docker dev environment --- docker/development/README.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docker/development/README.md diff --git a/docker/development/README.md b/docker/development/README.md new file mode 100644 index 00000000..2605b108 --- /dev/null +++ b/docker/development/README.md @@ -0,0 +1,27 @@ +# Running the development environment +1. Install Docker +2. Fill the values in .env +3. Run `./docker-compose-dev.sh up` to start the development environment +4. Connect to the development environment with your editor's remote SSH feature (see below) + +## Connecting with VSCode +1. Install the `Remote - SSH` plugin +2. Run `Remote-SSH: Connect to Host...` + * As the address, use `ubuntu@127.0.0.1:3002` where `3002` matches `DOCKER_DEV_SSH_PORT` in `.env` + * Use the password specified in `.env` as `DOCKER_DEV_SSH_PASSWORD` +3. Once connected, click `Open folder...` and select `/home/ubuntu/zeppelin` + +## Connecting with JetBrains Gateway +* TODO (basically the same as VSCode instructions though) + +## Starting the backend (bot + api) +These commands are run inside the dev container. You should be able to just open a terminal in your editor after connecting. +1. `cd ~/zeppelin/backend` +2. `npm ci` +3. `npm run migrate-dev` +4. `npm run watch` + +## Starting the dashboard +1. `cd ~/zeppelin/dashboard` +2. `npm ci` +3. `npm run watch-build` From ce2255b6b759edb2defb2d6ec8d6cddb2999581a Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 19:30:46 +0300 Subject: [PATCH 11/29] Simplify dev docker setup --- .env.example | 25 +++++++------ backend/src/api/auth.ts | 6 ++-- backend/src/api/start.ts | 2 +- backend/src/env.ts | 14 ++++---- dashboard/src/api.ts | 2 +- dashboard/src/auth.ts | 4 +-- dashboard/src/init-vue.ts | 2 +- docker-compose-dev.sh | 2 +- docker/development/README.md | 5 ++- docker/development/devenv/Dockerfile | 4 +-- docker/development/docker-compose.yml | 35 ++++-------------- docker/development/nginx/Dockerfile | 9 ++--- docker/development/nginx/default.conf | 52 +++++++++++---------------- 13 files changed, 65 insertions(+), 97 deletions(-) diff --git a/.env.example b/.env.example index 8d8052b9..88f8f5b5 100644 --- a/.env.example +++ b/.env.example @@ -1,36 +1,39 @@ -KEY=32_character_encryption_key +# 32 character encryption key +KEY= +# Values from the Discord developer portal CLIENT_ID= CLIENT_SECRET= BOT_TOKEN= -OAUTH_CALLBACK_URL= -DASHBOARD_DOMAIN= -API_DOMAIN= +DASHBOARD_URL=https://localhost:3300 +API_URL=https://localhost:3300/api + +# When using the Docker-based development environment, this is only used internally. The API will be available at localhost:DOCKER_DEV_WEB_PORT/api. API_PORT=3000 +# Only required if relevant feature is used +#PHISHERMAN_API_KEY= + # # DOCKER (DEVELOPMENT) # -DOCKER_WEB_PORT=443 +DOCKER_DEV_WEB_PORT=3300 # The MySQL database running in the container is exposed to the host on this port, # allowing access with database tools such as DBeaver -DOCKER_MYSQL_PORT=3001 +DOCKER_DEV_MYSQL_PORT=3001 # Password for the Zeppelin database user -DOCKER_MYSQL_PASSWORD= +DOCKER_DEV_MYSQL_PASSWORD= # Password for the MySQL root user -DOCKER_MYSQL_ROOT_PASSWORD= +DOCKER_DEV_MYSQL_ROOT_PASSWORD= # The development environment container has an SSH server that you can connect to. # This is the port that server is exposed to the host on. DOCKER_DEV_SSH_PORT=3002 DOCKER_DEV_SSH_PASSWORD=password -# Only required if relevant feature is used -#PHISHERMAN_API_KEY= - # # PRODUCTION # diff --git a/backend/src/api/auth.ts b/backend/src/api/auth.ts index 09029bda..ca947d8c 100644 --- a/backend/src/api/auth.ts +++ b/backend/src/api/auth.ts @@ -88,7 +88,7 @@ export function initAuth(app: express.Express) { tokenURL: "https://discord.com/api/oauth2/token", clientID: env.CLIENT_ID, clientSecret: env.CLIENT_SECRET, - callbackURL: env.OAUTH_CALLBACK_URL, + callbackURL: `${env.API_URL}/auth/oauth-callback`, scope: ["identify"], }, async (accessToken, refreshToken, profile, cb) => { @@ -117,9 +117,9 @@ export function initAuth(app: express.Express) { passport.authenticate("oauth2", { failureRedirect: "/", session: false }), (req: Request, res: Response) => { if (req.user && req.user.apiKey) { - res.redirect(`https://${env.DASHBOARD_DOMAIN}/login-callback/?apiKey=${req.user.apiKey}`); + res.redirect(`${env.DASHBOARD_URL}/login-callback/?apiKey=${req.user.apiKey}`); } else { - res.redirect(`https://${env.DASHBOARD_DOMAIN}/login-callback/?error=noAccess`); + res.redirect(`${env.DASHBOARD_URL}/login-callback/?error=noAccess`); } }, ); diff --git a/backend/src/api/start.ts b/backend/src/api/start.ts index 27d982ab..59a20da6 100644 --- a/backend/src/api/start.ts +++ b/backend/src/api/start.ts @@ -14,7 +14,7 @@ const app = express(); app.use( cors({ - origin: `https://${env.DASHBOARD_DOMAIN}`, + origin: env.DASHBOARD_URL, }), ); app.use( diff --git a/backend/src/env.ts b/backend/src/env.ts index b0eb3d9e..e786dd3c 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -11,17 +11,15 @@ const envType = z.object({ CLIENT_SECRET: z.string().length(32), BOT_TOKEN: z.string().min(50), - OAUTH_CALLBACK_URL: z.string().url(), - DASHBOARD_DOMAIN: z.string(), - API_DOMAIN: z.string(), + DASHBOARD_URL: z.string().url(), + API_URL: z.string().url(), + API_PORT: z.preprocess((v) => Number(v), z.number().min(1).max(65535)).default(3000), STAFF: z.preprocess((v) => String(v).split(","), z.array(z.string())).optional(), PHISHERMAN_API_KEY: z.string().optional(), - API_PORT: z.preprocess((v) => Number(v), z.number().min(1).max(65535)), - - DOCKER_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development + DOCKER_DEV_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development DB_HOST: z.string().optional().default("mysql"), DB_PORT: z @@ -42,6 +40,6 @@ if (fs.existsSync(envPath)) { export const env = envType.parse(toValidate); -if (env.DOCKER_MYSQL_PASSWORD && !env.DB_PASSWORD) { - env.DB_PASSWORD = env.DOCKER_MYSQL_PASSWORD; +if (env.DOCKER_DEV_MYSQL_PASSWORD && !env.DB_PASSWORD) { + env.DB_PASSWORD = env.DOCKER_DEV_MYSQL_PASSWORD; } diff --git a/dashboard/src/api.ts b/dashboard/src/api.ts index 6b75d1f8..f6cfd5e5 100644 --- a/dashboard/src/api.ts +++ b/dashboard/src/api.ts @@ -1,5 +1,5 @@ import { RootStore } from "./store"; -const apiUrl = `https://${process.env.API_DOMAIN}`; +const apiUrl = process.env.API_URL; type QueryParamObject = { [key: string]: string | null }; diff --git a/dashboard/src/auth.ts b/dashboard/src/auth.ts index e8c085d7..bb5c1b9e 100644 --- a/dashboard/src/auth.ts +++ b/dashboard/src/auth.ts @@ -11,7 +11,7 @@ const isAuthenticated = async () => { export const authGuard: NavigationGuard = async (to, from, next) => { if (await isAuthenticated()) return next(); - window.location.href = `https://${process.env.API_DOMAIN}/auth/login`; + window.location.href = `${process.env.API_URL}/auth/login`; }; export const loginCallbackGuard: NavigationGuard = async (to, from, next) => { @@ -25,5 +25,5 @@ export const loginCallbackGuard: NavigationGuard = async (to, from, next) => { export const authRedirectGuard: NavigationGuard = async (to, form, next) => { if (await isAuthenticated()) return next("/dashboard"); - window.location.href = `https://${process.env.API_DOMAIN}/auth/login`; + window.location.href = `${process.env.API_URL}/auth/login`; }; diff --git a/dashboard/src/init-vue.ts b/dashboard/src/init-vue.ts index 25b30969..614b5609 100644 --- a/dashboard/src/init-vue.ts +++ b/dashboard/src/init-vue.ts @@ -20,7 +20,7 @@ Vue.mixin({ return { get env() { return Object.freeze({ - API_URL: `https://${process.env.API_DOMAIN}`, + API_URL: process.env.API_URL, }); }, }; diff --git a/docker-compose-dev.sh b/docker-compose-dev.sh index 14af4e94..cd039f74 100755 --- a/docker-compose-dev.sh +++ b/docker-compose-dev.sh @@ -1,3 +1,3 @@ #!/bin/bash -DOCKER_UID="$(id -u)" DOCKER_GID="$(id -g)" docker-compose --env-file ./.env -f ./docker/development/docker-compose.yml "$@" +DOCKER_UID="$(id -u)" DOCKER_STAY_RUNNING=1 docker-compose --env-file ./.env -f ./docker/development/docker-compose.yml "$@" diff --git a/docker/development/README.md b/docker/development/README.md index 2605b108..ebbebf94 100644 --- a/docker/development/README.md +++ b/docker/development/README.md @@ -7,7 +7,7 @@ ## Connecting with VSCode 1. Install the `Remote - SSH` plugin 2. Run `Remote-SSH: Connect to Host...` - * As the address, use `ubuntu@127.0.0.1:3002` where `3002` matches `DOCKER_DEV_SSH_PORT` in `.env` + * As the address, use `ubuntu@127.0.0.1:3002` (where `3002` matches `DOCKER_DEV_SSH_PORT` in `.env`) * Use the password specified in `.env` as `DOCKER_DEV_SSH_PASSWORD` 3. Once connected, click `Open folder...` and select `/home/ubuntu/zeppelin` @@ -25,3 +25,6 @@ These commands are run inside the dev container. You should be able to just open 1. `cd ~/zeppelin/dashboard` 2. `npm ci` 3. `npm run watch-build` + +## Opening the dashboard +Browse to https://localhost:3300 to view the dashboard diff --git a/docker/development/devenv/Dockerfile b/docker/development/devenv/Dockerfile index ac278942..30963fee 100644 --- a/docker/development/devenv/Dockerfile +++ b/docker/development/devenv/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG DOCKER_UID +ARG DOCKER_UID=1000 ARG DOCKER_DEV_SSH_PASSWORD ENV DEBIAN_FRONTEND=noninteractive @@ -13,7 +13,7 @@ RUN apt-get install -y sudo git curl # Set up SSH access RUN apt-get install -y openssh-server iptables RUN mkdir /var/run/sshd -RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u "${DOCKER_UID}" ubuntu +RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u $DOCKER_UID ubuntu RUN echo "ubuntu:${DOCKER_DEV_SSH_PASSWORD}" | chpasswd # Set up proper permissions for volumes diff --git a/docker/development/docker-compose.yml b/docker/development/docker-compose.yml index 10c3caf1..7fcdd28a 100644 --- a/docker/development/docker-compose.yml +++ b/docker/development/docker-compose.yml @@ -9,54 +9,33 @@ services: build: context: ./nginx args: - API_DOMAIN: ${API_DOMAIN:?Missing API_DOMAIN} + DOCKER_DEV_WEB_PORT: ${DOCKER_DEV_WEB_PORT:?Missing DOCKER_DEV_WEB_PORT} API_PORT: ${API_PORT:?Missing API_PORT} - DASHBOARD_DOMAIN: ${DASHBOARD_DOMAIN:?Missing DASHBOARD_DOMAIN} ports: - - ${DOCKER_WEB_PORT:?Missing DOCKER_WEB_PORT}:443 + - "${DOCKER_DEV_WEB_PORT:?Missing DOCKER_DEV_WEB_PORT}:443" volumes: - ../../:/zeppelin mysql: image: mysql:8.0 environment: - MYSQL_ROOT_PASSWORD: ${DOCKER_MYSQL_ROOT_PASSWORD?:Missing DOCKER_MYSQL_ROOT_PASSWORD} + MYSQL_ROOT_PASSWORD: ${DOCKER_DEV_MYSQL_ROOT_PASSWORD?:Missing DOCKER_DEV_MYSQL_ROOT_PASSWORD} MYSQL_DATABASE: zeppelin MYSQL_USER: zeppelin - MYSQL_PASSWORD: ${DOCKER_MYSQL_PASSWORD?:Missing DOCKER_MYSQL_PASSWORD} + MYSQL_PASSWORD: ${DOCKER_DEV_MYSQL_PASSWORD?:Missing DOCKER_DEV_MYSQL_PASSWORD} ports: - - ${DOCKER_MYSQL_PORT:?Missing DOCKER_MYSQL_PORT}:3306 + - ${DOCKER_DEV_MYSQL_PORT:?Missing DOCKER_DEV_MYSQL_PORT}:3306 volumes: - mysql-data:/var/lib/mysql command: --authentication-policy=mysql_native_password -# -# backend: -# image: node:16 -# user: "${UID:?Missing UID}:${GID:?Missing GID}" -# working_dir: /zeppelin/backend -# restart: always -# depends_on: -# - mysql -# volumes: -# - ./:/zeppelin -# command: sh -c "npm run migrate-dev && npm run watch" -# -# dashboard: -# image: node:16 -# user: "${UID:?Missing UID}:${GID:?Missing GID}" -# working_dir: /zeppelin/dashboard -# restart: always -# volumes: -# - ./:/zeppelin -# command: sh -c "npm run watch-build" devenv: build: context: ./devenv args: - DOCKER_UID: ${DOCKER_UID:?Missing DOCKER_UID} - DOCKER_GID: ${DOCKER_GID:?Missing DOCKER_GID} DOCKER_DEV_SSH_PASSWORD: ${DOCKER_DEV_SSH_PASSWORD:?Missing DOCKER_DEV_SSH_PASSWORD} + DOCKER_UID: ${DOCKER_UID:?Missing DOCKER_UID} + DOCKER_STAY_RUNNING: ${DOCKER_STAY_RUNNING} ports: - "${DOCKER_DEV_SSH_PORT:?Missing DOCKER_DEV_SSH_PORT}:22" volumes: diff --git a/docker/development/nginx/Dockerfile b/docker/development/nginx/Dockerfile index 2f63ba2f..6df4e140 100644 --- a/docker/development/nginx/Dockerfile +++ b/docker/development/nginx/Dockerfile @@ -1,14 +1,11 @@ FROM nginx -ARG API_DOMAIN -ARG DASHBOARD_DOMAIN ARG API_PORT +ARG DOCKER_DEV_API_PORT +ARG DOCKER_DEV_DASHBOARD_PORT RUN apt-get update && apt-get install -y openssl -RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/api-cert.key -out /etc/ssl/certs/api-cert.pem -days 3650 -subj '/CN=*.${API_DOMAIN}' -nodes -RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/dashboard-cert.key -out /etc/ssl/certs/dashboard-cert.pem -days 3650 -subj '/CN=*.${DASHBOARD_DOMAIN}' -nodes +RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/localhost-cert.key -out /etc/ssl/certs/localhost-cert.pem -days 3650 -subj '/CN=localhost' -nodes COPY ./default.conf /etc/nginx/conf.d/default.conf -RUN sed -ir "s/_API_DOMAIN_/$(echo ${API_DOMAIN} | sed 's/\./\\./g')/g" /etc/nginx/conf.d/default.conf -RUN sed -ir "s/_DASHBOARD_DOMAIN_/$(echo ${DASHBOARD_DOMAIN} | sed 's/\./\\./g')/g" /etc/nginx/conf.d/default.conf RUN sed -ir "s/_API_PORT_/${API_PORT}/g" /etc/nginx/conf.d/default.conf diff --git a/docker/development/nginx/default.conf b/docker/development/nginx/default.conf index 3ff41fe0..9dc3db69 100644 --- a/docker/development/nginx/default.conf +++ b/docker/development/nginx/default.conf @@ -1,36 +1,7 @@ server { listen 443 ssl http2; listen [::]:443 ssl http2; - server_name _API_DOMAIN_; - - - location / { - # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable - set $backend_upstream "http://devenv:_API_PORT_"; - # Using a variable in proxy_pass also requires resolver to be set. - # This is the address of the internal docker compose DNS server. - resolver 127.0.0.11; - - proxy_pass $backend_upstream; - - client_max_body_size 200M; - } - - ssl_certificate /etc/ssl/certs/api-cert.pem; - ssl_certificate_key /etc/ssl/private/api-cert.key; - - ssl_session_timeout 1d; - ssl_session_cache shared:MozSSL:10m; - ssl_session_tickets off; - - ssl_protocols TLSv1.3; - ssl_prefer_server_ciphers off; -} - -server { - listen 443 ssl http2; - listen [::]:443 ssl http2; - server_name _DASHBOARD_DOMAIN_; + server_name localhost; root /zeppelin/dashboard/dist; @@ -39,8 +10,25 @@ server { try_files $uri $uri/ /index.html; } - ssl_certificate /etc/ssl/certs/dashboard-cert.pem; - ssl_certificate_key /etc/ssl/private/dashboard-cert.key; + # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable + set $backend_upstream "http://devenv:_API_PORT_"; + + location /api { + # Remove /api/ from the beginning when passing the path to the API process + rewrite /api(/.*)$ $1 break; + + # Using a variable in proxy_pass also requires resolver to be set. + # This is the address of the internal docker compose DNS server. + resolver 127.0.0.11; + + proxy_pass $backend_upstream$uri$is_args$args; + proxy_redirect off; + + client_max_body_size 200M; + } + + ssl_certificate /etc/ssl/certs/localhost-cert.pem; + ssl_certificate_key /etc/ssl/private/localhost-cert.key; ssl_session_timeout 1d; ssl_session_cache shared:MozSSL:10m; From d8b7d0cba56dba443383ca297f34694fb589ee02 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 19:33:56 +0300 Subject: [PATCH 12/29] Update Docker instructions --- docker/development/README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/development/README.md b/docker/development/README.md index ebbebf94..9a875434 100644 --- a/docker/development/README.md +++ b/docker/development/README.md @@ -1,8 +1,9 @@ # Running the development environment 1. Install Docker -2. Fill the values in .env -3. Run `./docker-compose-dev.sh up` to start the development environment -4. Connect to the development environment with your editor's remote SSH feature (see below) +2. Make a copy of `.env.example` called `.env` +3. Fill in the missing values in `.env` +4. Run `./docker-compose-dev.sh up` to start the development environment +5. Connect to the development environment with your editor's remote SSH feature (see below) ## Connecting with VSCode 1. Install the `Remote - SSH` plugin From 96101f267da3f5d7800a6173c8b15f7dd1273d3d Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 22:32:21 +0300 Subject: [PATCH 13/29] Add VSCode devcontainer support --- .devcontainer/devcontainer.json | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..d8b7643d --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,10 @@ +{ + "name": "Zeppelin Development", + + "dockerComposeFile": "../docker/development/docker-compose.yml", + "runArgs": ["--env-file", "../.env"], + + "service": "devenv", + "remoteUser": "ubuntu", + "workspaceFolder": "/home/ubuntu/zeppelin" +} From db84d80e745b6fa74fe05a135ee9d39ed6facce5 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 22:32:39 +0300 Subject: [PATCH 14/29] Update devenv instructions --- docker/development/README.md | 64 ++++++++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 13 deletions(-) diff --git a/docker/development/README.md b/docker/development/README.md index 9a875434..5ffc3a0e 100644 --- a/docker/development/README.md +++ b/docker/development/README.md @@ -1,31 +1,69 @@ -# Running the development environment +# Zeppelin development environment +Zeppelin's development environment runs entirely within a Docker container. +Below you can find instructions for setting up the environment and getting started with development! + +## Starting the development environment + +### Using VSCode devcontainers +1. Install Docker +2. Make a copy of `.env.example` called `.env` +3. Fill in the missing values in `.env` +4. In VSCode: Install the `Remote - Containers` plugin +5. In VSCode: Run `Remote-Containers: Open Folder in Container...` and select the Zeppelin folder + +### Using VSCode remote SSH plugin 1. Install Docker 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` 4. Run `./docker-compose-dev.sh up` to start the development environment -5. Connect to the development environment with your editor's remote SSH feature (see below) - -## Connecting with VSCode -1. Install the `Remote - SSH` plugin -2. Run `Remote-SSH: Connect to Host...` +5. In VSCode: Install the `Remote - SSH` plugin +6. In VSCode: Run `Remote-SSH: Connect to Host...` * As the address, use `ubuntu@127.0.0.1:3002` (where `3002` matches `DOCKER_DEV_SSH_PORT` in `.env`) * Use the password specified in `.env` as `DOCKER_DEV_SSH_PASSWORD` -3. Once connected, click `Open folder...` and select `/home/ubuntu/zeppelin` +7. In VSCode: Once connected, click `Open folder...` and select `/home/ubuntu/zeppelin` -## Connecting with JetBrains Gateway -* TODO (basically the same as VSCode instructions though) +### Using JetBrains Gateway +1. Install Docker +2. Make a copy of `.env.example` called `.env` +3. Fill in the missing values in `.env` +4. Run `./docker-compose-dev.sh up` to start the development environment +5. Choose `Connect via SSH` and create a new connection: + * Username: `ubuntu` + * Host: `127.0.0.1` + * Port: `3002` (matching the `DOCKER_DEV_SSH_PORT` value in `.env`) +6. Click `Check Connection and Continue` and enter the password specified in `.env` as `DOCKER_DEV_SSH_PASSWORD` when asked +7. In the next pane: + * IDE version: WebStorm, PHPStorm, or IntelliJ IDEA + * Project directory: `/home/ubuntu/zeppelin` +8. Click `Download and Start IDE` + +### Using any other IDE with SSH development support +1. Install Docker +2. Make a copy of `.env.example` called `.env` +3. Fill in the missing values in `.env` +4. Run `./docker-compose-dev.sh up` to start the development environment +5. Use the following credentials for connecting with your IDE: + * Host: `127.0.0.1` + * Port: `3002` (matching the `DOCKER_DEV_SSH_PORT` value in `.env`) + * Username: `ubuntu` + * Password: As specified in `.env` as `DOCKER_DEV_SSH_PASSWORD` + +## Starting the project + +### Starting the backend (bot + api) +These commands are run inside the dev container. You should be able to open a terminal in your IDE after connecting. -## Starting the backend (bot + api) -These commands are run inside the dev container. You should be able to just open a terminal in your editor after connecting. 1. `cd ~/zeppelin/backend` 2. `npm ci` 3. `npm run migrate-dev` 4. `npm run watch` -## Starting the dashboard +### Starting the dashboard +These commands are run inside the dev container. You should be able to open a terminal in your IDE after connecting. + 1. `cd ~/zeppelin/dashboard` 2. `npm ci` 3. `npm run watch-build` -## Opening the dashboard +### Opening the dashboard Browse to https://localhost:3300 to view the dashboard From 17fa8576094628bd33ee07a1aa6228f1eebc7300 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 26 Jun 2022 23:15:36 +0300 Subject: [PATCH 15/29] Remove docker-compose wrapper script; fixes to devcontainers --- .devcontainer/devcontainer.json | 3 +- .env.example | 3 ++ .../development/README.md => DEVELOPMENT.md | 8 ++-- README.md | 39 +------------------ docker-compose-dev.sh | 3 -- .../docker-compose.yml => docker-compose.yml | 11 +++--- docker/development/devenv/Dockerfile | 4 +- 7 files changed, 17 insertions(+), 54 deletions(-) rename docker/development/README.md => DEVELOPMENT.md (91%) delete mode 100755 docker-compose-dev.sh rename docker/development/docker-compose.yml => docker-compose.yml (85%) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d8b7643d..b44145a2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,8 +1,7 @@ { "name": "Zeppelin Development", - "dockerComposeFile": "../docker/development/docker-compose.yml", - "runArgs": ["--env-file", "../.env"], + "dockerComposeFile": "../docker-compose.yml", "service": "devenv", "remoteUser": "ubuntu", diff --git a/.env.example b/.env.example index 88f8f5b5..d98f2662 100644 --- a/.env.example +++ b/.env.example @@ -34,6 +34,9 @@ DOCKER_DEV_MYSQL_ROOT_PASSWORD= DOCKER_DEV_SSH_PORT=3002 DOCKER_DEV_SSH_PASSWORD=password +# If your user has a different UID than 1000, you might have to fill that in here to avoid permission issues +#DOCKER_DEV_UID=1000 + # # PRODUCTION # diff --git a/docker/development/README.md b/DEVELOPMENT.md similarity index 91% rename from docker/development/README.md rename to DEVELOPMENT.md index 5ffc3a0e..0edbbc88 100644 --- a/docker/development/README.md +++ b/DEVELOPMENT.md @@ -2,6 +2,8 @@ Zeppelin's development environment runs entirely within a Docker container. Below you can find instructions for setting up the environment and getting started with development! +πŸ‘‰ **No support is offered for self-hosting the bot!** πŸ‘ˆ + ## Starting the development environment ### Using VSCode devcontainers @@ -15,7 +17,7 @@ Below you can find instructions for setting up the environment and getting start 1. Install Docker 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` -4. Run `./docker-compose-dev.sh up` to start the development environment +4. Run `docker-compose up` to start the development environment 5. In VSCode: Install the `Remote - SSH` plugin 6. In VSCode: Run `Remote-SSH: Connect to Host...` * As the address, use `ubuntu@127.0.0.1:3002` (where `3002` matches `DOCKER_DEV_SSH_PORT` in `.env`) @@ -26,7 +28,7 @@ Below you can find instructions for setting up the environment and getting start 1. Install Docker 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` -4. Run `./docker-compose-dev.sh up` to start the development environment +4. Run `docker-compose up` to start the development environment 5. Choose `Connect via SSH` and create a new connection: * Username: `ubuntu` * Host: `127.0.0.1` @@ -41,7 +43,7 @@ Below you can find instructions for setting up the environment and getting start 1. Install Docker 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` -4. Run `./docker-compose-dev.sh up` to start the development environment +4. Run `docker-compose up` to start the development environment 5. Use the following credentials for connecting with your IDE: * Host: `127.0.0.1` * Port: `3002` (matching the `DOCKER_DEV_SSH_PORT` value in `.env`) diff --git a/README.md b/README.md index 02eae9ee..4d11bbe5 100644 --- a/README.md +++ b/README.md @@ -20,46 +20,9 @@ Zeppelin is a moderation bot for Discord, designed with large servers and reliab See https://zeppelin.gg/ for more details. ## Development -These instructions are intended for bot development only. - πŸ‘‰ **No support is offered for self-hosting the bot!** πŸ‘ˆ -### Running the bot -1. `cd backend` -2. `npm ci` -3. Make a copy of `bot.env.example` called `bot.env`, fill in the values -4. Run the desired start script: - * `npm run build` followed by `npm run start-bot-dev` to run the bot in a **development** environment - * `npm run build` followed by `npm run start-bot-prod` to run the bot in a **production** environment - * `npm run watch` to watch files and run the **bot and api both** in a **development** environment - with automatic restart on file changes -5. When testing, make sure you have your test server in the `allowed_guilds` table or the guild's config won't be loaded at all - -### Running the API server -1. `cd backend` -2. `npm ci` -3. Make a copy of `api.env.example` called `api.env`, fill in the values -4. Run the desired start script: - * `npm run build` followed by `npm run start-api-dev` to run the api in a **development** environment - * `npm run build` followed by `npm run start-api-prod` to run the api in a **production** environment - * `npm run watch` to watch files and run the **bot and api both** in a **development** environment - with automatic restart on file changes - -### Running the dashboard -1. `cd dashboard` -2. `npm ci` -3. Make a copy of `.env.example` called `.env`, fill in the values -4. Run the desired start script: - * `npm run build` compiles the dashboard's static files to `dist/` which can then be served with any web server - * `npm run watch` runs webpack's dev server that automatically reloads on changes - -### Notes -* Since we now use shared paths in `tsconfig.json`, the compiled files in `backend/dist/` have longer paths, e.g. - `backend/dist/backend/src/index.js` instead of `backend/dist/index.js`. This is because the compiled shared files - are placed in `backend/dist/shared`. -* The `backend/register-tsconfig-paths.js` module takes care of registering shared paths from `tsconfig.json` for - `ava` and compiled `.js` files -* To run the tests for the files in the `shared/` directory, you also need to run `npm ci` there +See [DEVELOPMENT.md](./DEVELOPMENT.md) for instructions on running the development environment! ### Config format example Configuration is stored in the database in the `configs` table diff --git a/docker-compose-dev.sh b/docker-compose-dev.sh deleted file mode 100755 index cd039f74..00000000 --- a/docker-compose-dev.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -DOCKER_UID="$(id -u)" DOCKER_STAY_RUNNING=1 docker-compose --env-file ./.env -f ./docker/development/docker-compose.yml "$@" diff --git a/docker/development/docker-compose.yml b/docker-compose.yml similarity index 85% rename from docker/development/docker-compose.yml rename to docker-compose.yml index 7fcdd28a..1c836bd2 100644 --- a/docker/development/docker-compose.yml +++ b/docker-compose.yml @@ -7,14 +7,14 @@ volumes: services: nginx: build: - context: ./nginx + context: ./docker/development/nginx args: DOCKER_DEV_WEB_PORT: ${DOCKER_DEV_WEB_PORT:?Missing DOCKER_DEV_WEB_PORT} API_PORT: ${API_PORT:?Missing API_PORT} ports: - "${DOCKER_DEV_WEB_PORT:?Missing DOCKER_DEV_WEB_PORT}:443" volumes: - - ../../:/zeppelin + - ./:/zeppelin mysql: image: mysql:8.0 @@ -31,15 +31,14 @@ services: devenv: build: - context: ./devenv + context: ./docker/development/devenv args: DOCKER_DEV_SSH_PASSWORD: ${DOCKER_DEV_SSH_PASSWORD:?Missing DOCKER_DEV_SSH_PASSWORD} - DOCKER_UID: ${DOCKER_UID:?Missing DOCKER_UID} - DOCKER_STAY_RUNNING: ${DOCKER_STAY_RUNNING} + DOCKER_DEV_UID: ${DOCKER_DEV_UID:-1000} ports: - "${DOCKER_DEV_SSH_PORT:?Missing DOCKER_DEV_SSH_PORT}:22" volumes: - - ../../:/home/ubuntu/zeppelin + - ./:/home/ubuntu/zeppelin - ~/.ssh:/home/ubuntu/.ssh - vscode-remote:/home/ubuntu/.vscode-remote - vscode-server:/home/ubuntu/.vscode-server diff --git a/docker/development/devenv/Dockerfile b/docker/development/devenv/Dockerfile index 30963fee..3b9d6276 100644 --- a/docker/development/devenv/Dockerfile +++ b/docker/development/devenv/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG DOCKER_UID=1000 +ARG DOCKER_DEV_UID ARG DOCKER_DEV_SSH_PASSWORD ENV DEBIAN_FRONTEND=noninteractive @@ -13,7 +13,7 @@ RUN apt-get install -y sudo git curl # Set up SSH access RUN apt-get install -y openssh-server iptables RUN mkdir /var/run/sshd -RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u $DOCKER_UID ubuntu +RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u $DOCKER_DEV_UID ubuntu RUN echo "ubuntu:${DOCKER_DEV_SSH_PASSWORD}" | chpasswd # Set up proper permissions for volumes From 91f54424ed0262bde612dc26801763837ec4a74a Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sat, 16 Jul 2022 22:16:34 +0300 Subject: [PATCH 16/29] Early work on prod container --- .devcontainer/devcontainer.json | 2 +- .env.example | 12 ++-- backend/package.json | 4 +- backend/src/env.ts | 9 ++- ...pose.yml => docker-compose.development.yml | 1 + docker-compose.production.yml | 63 +++++++++++++++++++ docker/production/nginx/Dockerfile | 11 ++++ docker/production/nginx/default.conf | 39 ++++++++++++ 8 files changed, 132 insertions(+), 9 deletions(-) rename docker-compose.yml => docker-compose.development.yml (98%) create mode 100644 docker-compose.production.yml create mode 100644 docker/production/nginx/Dockerfile create mode 100644 docker/production/nginx/default.conf diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b44145a2..0d15eb29 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,7 +1,7 @@ { "name": "Zeppelin Development", - "dockerComposeFile": "../docker-compose.yml", + "dockerComposeFile": "../docker-compose.development.yml", "service": "devenv", "remoteUser": "ubuntu", diff --git a/.env.example b/.env.example index d98f2662..d2d2a2b2 100644 --- a/.env.example +++ b/.env.example @@ -38,12 +38,16 @@ DOCKER_DEV_SSH_PASSWORD=password #DOCKER_DEV_UID=1000 # -# PRODUCTION +# DOCKER (PRODUCTION) # -# In production, the newest code is pulled from a repository -# Specify that repository URL here -#PRODUCTION_REPOSITORY=https://github.com/ZeppelinBot/Zeppelin.git +DOCKER_PROD_DOMAIN= +DOCKER_PROD_WEB_PORT=443 +DOCKER_PROD_MYSQL_PORT=3001 +# Password for the Zeppelin database user +DOCKER_PROD_MYSQL_PASSWORD= +# Password for the MySQL root user +DOCKER_PROD_MYSQL_ROOT_PASSWORD= # You only need to set these if you're running an external database. # In a standard setup, the database is run in a docker container. diff --git a/backend/package.json b/backend/package.json index 4761ec2e..6be4713d 100644 --- a/backend/package.json +++ b/backend/package.json @@ -14,9 +14,9 @@ "start-api-prod": "cross-env NODE_ENV=production node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --stack-trace-limit=30 dist/backend/src/api/index.js", "watch-api": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-api-dev\"", "typeorm": "node -r ./register-tsconfig-paths.js ./node_modules/typeorm/cli.js", - "migrate-prod": "npm run typeorm -- migration:run", + "migrate-prod": "cross-env NODE_ENV=production npm run typeorm -- migration:run", "migrate-dev": "npm run build && npm run typeorm -- migration:run", - "migrate-rollback-prod": "npm run typeorm -- migration:revert", + "migrate-rollback-prod": "cross-env NODE_ENV=production npm run typeorm -- migration:revert", "migrate-rollback-dev": "npm run build && npm run typeorm -- migration:revert", "test": "npm run build && npm run run-tests", "run-tests": "ava", diff --git a/backend/src/env.ts b/backend/src/env.ts index e786dd3c..cd453f39 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -20,6 +20,7 @@ const envType = z.object({ PHISHERMAN_API_KEY: z.string().optional(), DOCKER_DEV_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development + DOCKER_PROD_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in production DB_HOST: z.string().optional().default("mysql"), DB_PORT: z @@ -40,6 +41,10 @@ if (fs.existsSync(envPath)) { export const env = envType.parse(toValidate); -if (env.DOCKER_DEV_MYSQL_PASSWORD && !env.DB_PASSWORD) { - env.DB_PASSWORD = env.DOCKER_DEV_MYSQL_PASSWORD; +if (!env.DB_PASSWORD) { + if (process.env.NODE_ENV === "production" && env.DOCKER_PROD_MYSQL_PASSWORD) { + env.DB_PASSWORD = env.DOCKER_PROD_MYSQL_PASSWORD; + } else if (env.DOCKER_DEV_MYSQL_PASSWORD) { + env.DB_PASSWORD = env.DOCKER_DEV_MYSQL_PASSWORD; + } } diff --git a/docker-compose.yml b/docker-compose.development.yml similarity index 98% rename from docker-compose.yml rename to docker-compose.development.yml index 1c836bd2..4a28033e 100644 --- a/docker-compose.yml +++ b/docker-compose.development.yml @@ -1,4 +1,5 @@ version: '3' +name: zeppelin-dev volumes: mysql-data: {} vscode-remote: {} diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 00000000..1b03bd00 --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,63 @@ +version: '3' +name: zeppelin-prod +volumes: + mysql-data: {} +services: + nginx: + build: + context: ./docker/production/nginx + args: + API_PORT: ${API_PORT:?Missing API_PORT} + DOCKER_PROD_DOMAIN: ${DOCKER_PROD_DOMAIN:?Missing DOCKER_PROD_DOMAIN} + ports: + - "${DOCKER_PROD_WEB_PORT:?Missing DOCKER_PROD_WEB_PORT}:443" + volumes: + - ./:/zeppelin + + mysql: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: ${DOCKER_PROD_MYSQL_ROOT_PASSWORD?:Missing DOCKER_PROD_MYSQL_ROOT_PASSWORD} + MYSQL_DATABASE: zeppelin + MYSQL_USER: zeppelin + MYSQL_PASSWORD: ${DOCKER_PROD_MYSQL_PASSWORD?:Missing DOCKER_PROD_MYSQL_PASSWORD} + ports: + - ${DOCKER_PROD_MYSQL_PORT:?Missing DOCKER_PROD_MYSQL_PORT}:3306 + volumes: + - mysql-data:/var/lib/mysql + command: --authentication-policy=mysql_native_password + + prepare_backend: + image: node:16.16 + depends_on: + - mysql + volumes: + - ./:/zeppelin + user: node + command: |- + bash -c "cd /zeppelin/backend && npm ci && npm run build && npm run migrate-prod" + + api: + image: node:16.16 + restart: on-failure + depends_on: + - prepare_backend + volumes: + - ./:/zeppelin + + # Wait for the build_backend container to finish before starting the bot + # See: https://github.com/docker/compose/issues/5007#issuecomment-335815508 + user: node + command: |- + bash -c ' \ + while ping -c1 prepare_backend &>/dev/null; do sleep 1; done; \ + cd /zeppelin/backend && npm run start-api-prod \ + ' + + build_dashboard: + image: node:16.16 + volumes: + - ./:/zeppelin + user: node + command: |- + bash -c "cd /zeppelin/dashboard && npm ci && npm run build" diff --git a/docker/production/nginx/Dockerfile b/docker/production/nginx/Dockerfile new file mode 100644 index 00000000..47df0a26 --- /dev/null +++ b/docker/production/nginx/Dockerfile @@ -0,0 +1,11 @@ +FROM nginx + +ARG API_PORT +ARG DOCKER_PROD_DOMAIN + +RUN apt-get update && apt-get install -y openssl +RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/zeppelin-self-signed-cert.key -out /etc/ssl/certs/zeppelin-self-signed-cert.pem -days 3650 -subj "/CN=${DOCKER_PROD_DOMAIN}" -nodes + +COPY ./default.conf /etc/nginx/conf.d/default.conf +RUN sed -ir "s/_API_PORT_/${API_PORT}/g" /etc/nginx/conf.d/default.conf +RUN sed -ir "s/_DOCKER_PROD_DOMAIN_/${DOCKER_PROD_DOMAIN}/g" /etc/nginx/conf.d/default.conf diff --git a/docker/production/nginx/default.conf b/docker/production/nginx/default.conf new file mode 100644 index 00000000..2fcc93c0 --- /dev/null +++ b/docker/production/nginx/default.conf @@ -0,0 +1,39 @@ +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name _DOCKER_PROD_DOMAIN_; + + root /zeppelin/dashboard/dist; + + location / { + index index.html; + try_files $uri $uri/ /index.html; + } + + # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable + set $backend_upstream "http://api:_API_PORT_"; + + location /api { + # Remove /api/ from the beginning when passing the path to the API process + rewrite /api(/.*)$ $1 break; + + # Using a variable in proxy_pass also requires resolver to be set. + # This is the address of the internal docker compose DNS server. + resolver 127.0.0.11; + + proxy_pass $backend_upstream$uri$is_args$args; + proxy_redirect off; + + client_max_body_size 200M; + } + + ssl_certificate /etc/ssl/certs/zeppelin-self-signed-cert.pem; + ssl_certificate_key /etc/ssl/private/zeppelin-self-signed-cert.key; + + ssl_session_timeout 1d; + ssl_session_cache shared:MozSSL:10m; + ssl_session_tickets off; + + ssl_protocols TLSv1.3; + ssl_prefer_server_ciphers off; +} From f7fede47bd49ac48731b2ff8d2f976efe6aa6b32 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sat, 6 Aug 2022 21:31:13 +0300 Subject: [PATCH 17/29] refactor: use .env STAFF instead of global config owners for global commands --- .env.example | 3 +++ backend/src/pluginUtils.ts | 5 +++-- .../src/plugins/BotControl/commands/AddDashboardUserCmd.ts | 4 ++-- .../plugins/BotControl/commands/AddServerFromInviteCmd.ts | 2 +- backend/src/plugins/BotControl/commands/AllowServerCmd.ts | 4 ++-- .../src/plugins/BotControl/commands/ChannelToServerCmd.ts | 4 ++-- backend/src/plugins/BotControl/commands/DisallowServerCmd.ts | 4 ++-- backend/src/plugins/BotControl/commands/LeaveServerCmd.ts | 4 ++-- .../src/plugins/BotControl/commands/ListDashboardPermsCmd.ts | 2 +- .../src/plugins/BotControl/commands/ListDashboardUsersCmd.ts | 2 +- .../plugins/BotControl/commands/ReloadGlobalPluginsCmd.ts | 4 ++-- backend/src/plugins/BotControl/commands/ReloadServerCmd.ts | 4 ++-- .../plugins/BotControl/commands/RemoveDashboardUserCmd.ts | 4 ++-- backend/src/plugins/BotControl/commands/ServersCmd.ts | 4 ++-- 14 files changed, 27 insertions(+), 23 deletions(-) diff --git a/.env.example b/.env.example index d2d2a2b2..a794dc46 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,9 @@ BOT_TOKEN= DASHBOARD_URL=https://localhost:3300 API_URL=https://localhost:3300/api +# Comma-separated list of user IDs who should have access to the bot's global commands +STAFF= + # When using the Docker-based development environment, this is only used internally. The API will be available at localhost:DOCKER_DEV_WEB_PORT/api. API_PORT=3000 diff --git a/backend/src/pluginUtils.ts b/backend/src/pluginUtils.ts index 031d1fa7..17cd7631 100644 --- a/backend/src/pluginUtils.ts +++ b/backend/src/pluginUtils.ts @@ -14,6 +14,7 @@ import { TZeppelinKnub } from "./types"; import { deepKeyIntersect, errorMessage, successMessage, tDeepPartial, tNullable } from "./utils"; import { Tail } from "./utils/typeUtils"; import { decodeAndValidateStrict, StrictValidationError, validate } from "./validatorUtils"; +import { isStaff } from "./staff"; const { getMemberLevel } = helpers; @@ -242,8 +243,8 @@ export function isOwner(pluginData: AnyPluginData, userId: string) { return owners.includes(userId); } -export const isOwnerPreFilter = (_, context: CommandContext) => { - return isOwner(context.pluginData, context.message.author.id); +export const isStaffPreFilter = (_, context: CommandContext) => { + return isStaff(context.message.author.id); }; type AnyFn = (...args: any[]) => any; diff --git a/backend/src/plugins/BotControl/commands/AddDashboardUserCmd.ts b/backend/src/plugins/BotControl/commands/AddDashboardUserCmd.ts index b417fec6..531016a3 100644 --- a/backend/src/plugins/BotControl/commands/AddDashboardUserCmd.ts +++ b/backend/src/plugins/BotControl/commands/AddDashboardUserCmd.ts @@ -1,14 +1,14 @@ import { ApiPermissions } from "@shared/apiPermissions"; import { TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { botControlCmd } from "../types"; export const AddDashboardUserCmd = botControlCmd({ trigger: ["add_dashboard_user"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { diff --git a/backend/src/plugins/BotControl/commands/AddServerFromInviteCmd.ts b/backend/src/plugins/BotControl/commands/AddServerFromInviteCmd.ts index 5bdb2ace..1f29e822 100644 --- a/backend/src/plugins/BotControl/commands/AddServerFromInviteCmd.ts +++ b/backend/src/plugins/BotControl/commands/AddServerFromInviteCmd.ts @@ -1,7 +1,7 @@ import { ApiPermissions } from "@shared/apiPermissions"; import { TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { DBDateFormat, isGuildInvite, isSnowflake, resolveInvite } from "../../../utils"; import { botControlCmd } from "../types"; import moment from "moment-timezone"; diff --git a/backend/src/plugins/BotControl/commands/AllowServerCmd.ts b/backend/src/plugins/BotControl/commands/AllowServerCmd.ts index 9e786b71..1aa665b5 100644 --- a/backend/src/plugins/BotControl/commands/AllowServerCmd.ts +++ b/backend/src/plugins/BotControl/commands/AllowServerCmd.ts @@ -1,7 +1,7 @@ import { ApiPermissions } from "@shared/apiPermissions"; import { TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { DBDateFormat, isSnowflake } from "../../../utils"; import { botControlCmd } from "../types"; import moment from "moment-timezone"; @@ -10,7 +10,7 @@ export const AllowServerCmd = botControlCmd({ trigger: ["allow_server", "allowserver", "add_server", "addserver"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { diff --git a/backend/src/plugins/BotControl/commands/ChannelToServerCmd.ts b/backend/src/plugins/BotControl/commands/ChannelToServerCmd.ts index 5e9dcbb8..49f0ff73 100644 --- a/backend/src/plugins/BotControl/commands/ChannelToServerCmd.ts +++ b/backend/src/plugins/BotControl/commands/ChannelToServerCmd.ts @@ -1,6 +1,6 @@ import { Guild, GuildChannel, TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { GuildInvite, isGuildInvite, resolveInvite, verboseUserMention } from "../../../utils"; import { botControlCmd } from "../types"; import { isEligible } from "../functions/isEligible"; @@ -9,7 +9,7 @@ export const ChannelToServerCmd = botControlCmd({ trigger: ["channel_to_server", "channel2server"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { diff --git a/backend/src/plugins/BotControl/commands/DisallowServerCmd.ts b/backend/src/plugins/BotControl/commands/DisallowServerCmd.ts index 1a9cf64a..c7067341 100644 --- a/backend/src/plugins/BotControl/commands/DisallowServerCmd.ts +++ b/backend/src/plugins/BotControl/commands/DisallowServerCmd.ts @@ -1,6 +1,6 @@ import { Snowflake, TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { noop } from "../../../utils"; import { botControlCmd } from "../types"; @@ -8,7 +8,7 @@ export const DisallowServerCmd = botControlCmd({ trigger: ["disallow_server", "disallowserver", "remove_server", "removeserver"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { diff --git a/backend/src/plugins/BotControl/commands/LeaveServerCmd.ts b/backend/src/plugins/BotControl/commands/LeaveServerCmd.ts index b502c08f..20d0b14b 100644 --- a/backend/src/plugins/BotControl/commands/LeaveServerCmd.ts +++ b/backend/src/plugins/BotControl/commands/LeaveServerCmd.ts @@ -1,13 +1,13 @@ import { Snowflake, TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { botControlCmd } from "../types"; export const LeaveServerCmd = botControlCmd({ trigger: ["leave_server", "leave_guild"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { diff --git a/backend/src/plugins/BotControl/commands/ListDashboardPermsCmd.ts b/backend/src/plugins/BotControl/commands/ListDashboardPermsCmd.ts index dcfb166c..590522f1 100644 --- a/backend/src/plugins/BotControl/commands/ListDashboardPermsCmd.ts +++ b/backend/src/plugins/BotControl/commands/ListDashboardPermsCmd.ts @@ -2,7 +2,7 @@ import { TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; import { AllowedGuild } from "../../../data/entities/AllowedGuild"; import { ApiPermissionAssignment } from "../../../data/entities/ApiPermissionAssignment"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { resolveUser } from "../../../utils"; import { botControlCmd } from "../types"; diff --git a/backend/src/plugins/BotControl/commands/ListDashboardUsersCmd.ts b/backend/src/plugins/BotControl/commands/ListDashboardUsersCmd.ts index a97efca6..90fee6ff 100644 --- a/backend/src/plugins/BotControl/commands/ListDashboardUsersCmd.ts +++ b/backend/src/plugins/BotControl/commands/ListDashboardUsersCmd.ts @@ -1,6 +1,6 @@ import { TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { resolveUser } from "../../../utils"; import { botControlCmd } from "../types"; diff --git a/backend/src/plugins/BotControl/commands/ReloadGlobalPluginsCmd.ts b/backend/src/plugins/BotControl/commands/ReloadGlobalPluginsCmd.ts index 1b3629ce..0cd4ec0b 100644 --- a/backend/src/plugins/BotControl/commands/ReloadGlobalPluginsCmd.ts +++ b/backend/src/plugins/BotControl/commands/ReloadGlobalPluginsCmd.ts @@ -1,5 +1,5 @@ import { TextChannel } from "discord.js"; -import { isOwnerPreFilter } from "../../../pluginUtils"; +import { isStaffPreFilter } from "../../../pluginUtils"; import { getActiveReload, setActiveReload } from "../activeReload"; import { botControlCmd } from "../types"; @@ -7,7 +7,7 @@ export const ReloadGlobalPluginsCmd = botControlCmd({ trigger: "bot_reload_global_plugins", permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, async run({ pluginData, message }) { diff --git a/backend/src/plugins/BotControl/commands/ReloadServerCmd.ts b/backend/src/plugins/BotControl/commands/ReloadServerCmd.ts index 88193fba..645e4cf4 100644 --- a/backend/src/plugins/BotControl/commands/ReloadServerCmd.ts +++ b/backend/src/plugins/BotControl/commands/ReloadServerCmd.ts @@ -1,13 +1,13 @@ import { Snowflake, TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { botControlCmd } from "../types"; export const ReloadServerCmd = botControlCmd({ trigger: ["reload_server", "reload_guild"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { diff --git a/backend/src/plugins/BotControl/commands/RemoveDashboardUserCmd.ts b/backend/src/plugins/BotControl/commands/RemoveDashboardUserCmd.ts index dff4928d..d5dd674b 100644 --- a/backend/src/plugins/BotControl/commands/RemoveDashboardUserCmd.ts +++ b/backend/src/plugins/BotControl/commands/RemoveDashboardUserCmd.ts @@ -1,13 +1,13 @@ import { TextChannel } from "discord.js"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; +import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils"; import { botControlCmd } from "../types"; export const RemoveDashboardUserCmd = botControlCmd({ trigger: ["remove_dashboard_user"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { diff --git a/backend/src/plugins/BotControl/commands/ServersCmd.ts b/backend/src/plugins/BotControl/commands/ServersCmd.ts index f8c91b61..cee486d2 100644 --- a/backend/src/plugins/BotControl/commands/ServersCmd.ts +++ b/backend/src/plugins/BotControl/commands/ServersCmd.ts @@ -1,7 +1,7 @@ import { TextChannel } from "discord.js"; import escapeStringRegexp from "escape-string-regexp"; import { commandTypeHelpers as ct } from "../../../commandTypes"; -import { isOwnerPreFilter } from "../../../pluginUtils"; +import { isStaffPreFilter } from "../../../pluginUtils"; import { createChunkedMessage, getUser, sorter } from "../../../utils"; import { botControlCmd } from "../types"; @@ -9,7 +9,7 @@ export const ServersCmd = botControlCmd({ trigger: ["servers", "guilds"], permission: null, config: { - preFilters: [isOwnerPreFilter], + preFilters: [isStaffPreFilter], }, signature: { From 4a5e8ded75e952e7dfff31ceb1a909f4af0ed8fc Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sat, 6 Aug 2022 22:12:40 +0300 Subject: [PATCH 18/29] feat: add DEFAULT_ALLOWED_SERVERS .env value --- .env.example | 3 +++ backend/src/data/AllowedGuilds.ts | 1 + backend/src/env.ts | 2 ++ .../GuildAccessMonitorPlugin.ts | 15 ++++++++++++++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index a794dc46..cc76ef84 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,9 @@ API_URL=https://localhost:3300/api # Comma-separated list of user IDs who should have access to the bot's global commands STAFF= +# A comma-separated list of server IDs that should be allowed by default +DEFAULT_ALLOWED_SERVERS= + # When using the Docker-based development environment, this is only used internally. The API will be available at localhost:DOCKER_DEV_WEB_PORT/api. API_PORT=3000 diff --git a/backend/src/data/AllowedGuilds.ts b/backend/src/data/AllowedGuilds.ts index 747dbb87..a9d86220 100644 --- a/backend/src/data/AllowedGuilds.ts +++ b/backend/src/data/AllowedGuilds.ts @@ -4,6 +4,7 @@ import { BaseRepository } from "./BaseRepository"; import { AllowedGuild } from "./entities/AllowedGuild"; import moment from "moment-timezone"; import { DBDateFormat } from "../utils"; +import { env } from "../env"; export class AllowedGuilds extends BaseRepository { private allowedGuilds: Repository; diff --git a/backend/src/env.ts b/backend/src/env.ts index cd453f39..5a607e45 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -17,6 +17,8 @@ const envType = z.object({ STAFF: z.preprocess((v) => String(v).split(","), z.array(z.string())).optional(), + DEFAULT_ALLOWED_SERVERS: z.preprocess((v) => String(v).split(","), z.array(z.string())).optional(), + PHISHERMAN_API_KEY: z.string().optional(), DOCKER_DEV_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development diff --git a/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts b/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts index d181ad7a..711a9154 100644 --- a/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts +++ b/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts @@ -3,6 +3,8 @@ import * as t from "io-ts"; import { BasePluginType, GlobalPluginData, typedGlobalEventListener } from "knub"; import { AllowedGuilds } from "../../data/AllowedGuilds"; import { zeppelinGlobalPlugin } from "../ZeppelinPluginBlueprint"; +import { env } from "../../env"; +import { Configs } from "../../data/Configs"; interface GuildAccessMonitorPluginType extends BasePluginType { config: {}; @@ -35,8 +37,19 @@ export const GuildAccessMonitorPlugin = zeppelinGlobalPlugin Date: Sat, 6 Aug 2022 22:13:07 +0300 Subject: [PATCH 19/29] debug: temporarily disable server auto-leaving This is to avoid any potential funny business during the upcoming server migration. --- .../plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts b/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts index 711a9154..80da1ebd 100644 --- a/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts +++ b/backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts @@ -17,7 +17,7 @@ async function checkGuild(pluginData: GlobalPluginData Date: Sat, 6 Aug 2022 22:18:21 +0300 Subject: [PATCH 20/29] feat: more work on prod docker setup --- .env.example | 2 + DEVELOPMENT.md | 2 +- MANAGEMENT.md | 34 ++++++++++++ PRODUCTION.md | 32 +++++++++++ README.md | 53 +++---------------- docker-compose.development.yml | 3 +- docker-compose.production.yml | 23 ++++---- docker/development/data/mysql/.gitignore | 2 + .../production/config/mysql.conf.d/.gitignore | 2 + docker/production/data/mysql/.gitignore | 2 + docker/production/start-api.sh | 13 +++++ docker/production/start-bot.sh | 13 +++++ update.sh | 9 +++- 13 files changed, 129 insertions(+), 61 deletions(-) create mode 100644 MANAGEMENT.md create mode 100644 PRODUCTION.md create mode 100644 docker/development/data/mysql/.gitignore create mode 100644 docker/production/config/mysql.conf.d/.gitignore create mode 100644 docker/production/data/mysql/.gitignore create mode 100755 docker/production/start-api.sh create mode 100644 docker/production/start-bot.sh diff --git a/.env.example b/.env.example index cc76ef84..4494f3b0 100644 --- a/.env.example +++ b/.env.example @@ -49,6 +49,8 @@ DOCKER_DEV_SSH_PASSWORD=password DOCKER_PROD_DOMAIN= DOCKER_PROD_WEB_PORT=443 +# The MySQL database running in the container is exposed to the host on this port, +# allowing access with database tools such as DBeaver DOCKER_PROD_MYSQL_PORT=3001 # Password for the Zeppelin database user DOCKER_PROD_MYSQL_PASSWORD= diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 0edbbc88..90f2c2ca 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -2,7 +2,7 @@ Zeppelin's development environment runs entirely within a Docker container. Below you can find instructions for setting up the environment and getting started with development! -πŸ‘‰ **No support is offered for self-hosting the bot!** πŸ‘ˆ +**Note:** If you'd just like to run the bot for your own server, see πŸ‘‰ **[PRODUCTION.md](./PRODUCTION.md)** πŸ‘ˆ ## Starting the development environment diff --git a/MANAGEMENT.md b/MANAGEMENT.md new file mode 100644 index 00000000..a6d12ef5 --- /dev/null +++ b/MANAGEMENT.md @@ -0,0 +1,34 @@ +# Management +After starting Zeppelin -- either in the [development](./DEVELOPMENT.md) or [production](./PRODUCTION.md) environment -- you have several tools available to manage it. + +## Note +Make sure to add yourself to the list of staff members (`STAFF`) in `.env` and allow at least one server by default (`DEFAULT_ALLOWED_SERVERS`). Then, invite the bot to the server. + +In all examples below, `@Bot` refers to a user mention of the bot user. Make sure to run the commands on a server with the bot, in a channel that the bot can see. + +In the command parameters, `` refers to a required parameter (don't include the `< >` symbols) and `[this]` refers to an optional parameter (don't include the `[ ]` symbols). `` refers to being able to list multiple values, e.g. `value1 value2 value3`. + +## Allow a server to invite the bot +Run the following command: +``` +@Bot allow_server [userId] +``` +When specifying a user ID, that user will be given "Bot manager" level access to the server's dashboard, allowing them to manage access for other users. + +## Disallow a server +Run the following command: +``` +@Bot disallow_server +``` + +## Grant access to a server's dashboard +Run the following command: +``` +@Bot add_dashboard_user +``` + +## Remove access to a server's dashboard +Run the following command: +``` +@Bot remove_dashboard_user +``` diff --git a/PRODUCTION.md b/PRODUCTION.md new file mode 100644 index 00000000..a45bf50d --- /dev/null +++ b/PRODUCTION.md @@ -0,0 +1,32 @@ +# Zeppelin production environment +Zeppelin's production environment - that is, the **bot, API, and dashboard** - uses Docker. + +## Starting the production environment +1. Install Docker on the machine running the bot +2. Make a copy of `.env.example` called `.env` +3. Fill in the missing values in `.env` +4. Run `docker-compose -f docker-compose.production.yml -d up` + +## Updating the bot + +### One-click script +If you've downloaded the bot's files by cloning the git repository, you can use `update.sh` to update the bot. + +### Manual instructions +1. Shut the bot down: `docker-compose -f docker-compose.production.yml down` +2. Update the files (e.g. `git pull`) +3. Start the bot again: `docker-compose -f docker-compose.production.yml -d up` + +### Ephemeral hotfixes +If you need to make a hotfix to the bot's source files directly on the server: +1. Shut the bot down: `docker-compose -f docker-compose.production.yml down` +2. Make your edits +3. Start the bot again: `docker-compose -f docker-compose.production.yml -d up` + +Note that you can't edit the compiled files directly as they're overwritten when the environment starts. +Only edit files in `/backend/src`, `/shared/src`, and `/dashboard/src`. + +Make sure to revert any hotfixes before updating the bot normally. + +## View logs +To view real-time logs, run `docker-compose -f docker-compose.production.yml -t logs` diff --git a/README.md b/README.md index 4d11bbe5..52f60f07 100644 --- a/README.md +++ b/README.md @@ -19,52 +19,15 @@ Zeppelin is a moderation bot for Discord, designed with large servers and reliab See https://zeppelin.gg/ for more details. +## Usage documentation +For information on how to use the bot, see https://zeppelin.gg/docs + ## Development -πŸ‘‰ **No support is offered for self-hosting the bot!** πŸ‘ˆ +See [DEVELOPMENT.md](./DEVELOPMENT.md) for instructions on running the development environment. -See [DEVELOPMENT.md](./DEVELOPMENT.md) for instructions on running the development environment! +Once you have the environment up and running, see [MANAGEMENT.md](./MANAGEMENT.md) for how to manage your bot. -### Config format example -Configuration is stored in the database in the `configs` table +## Production +See [PRODUCTION.md](./PRODUCTION.md) for instructions on how to run the bot in production. -```yml -prefix: '!' - -# role id: level -levels: - "12345678": 100 # Example admin - "98765432": 50 # Example mod - -plugins: - mod_plugin: - config: - kick_message: 'You have been kicked' - can_kick: false - overrides: - - level: '>=50' - config: - can_kick: true - - level: '>=100' - config: - kick_message: 'You have been kicked by an admin' - - other_plugin: - config: - categories: - mycategory: - opt: "something" - othercategory: - enabled: false - opt: "hello" - overrides: - - level: '>=50' - config: - categories: - mycategory: - enabled: false - - channel: '1234' - config: - categories: - othercategory: - enabled: true -``` +Once you have the environment up and running, see [MANAGEMENT.md](./MANAGEMENT.md) for how to manage your bot. diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 4a28033e..d64aa393 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -1,7 +1,6 @@ version: '3' name: zeppelin-dev volumes: - mysql-data: {} vscode-remote: {} vscode-server: {} jetbrains-data: {} @@ -27,7 +26,7 @@ services: ports: - ${DOCKER_DEV_MYSQL_PORT:?Missing DOCKER_DEV_MYSQL_PORT}:3306 volumes: - - mysql-data:/var/lib/mysql + - ./docker/development/data/mysql:/var/lib/mysql command: --authentication-policy=mysql_native_password devenv: diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 1b03bd00..109cb3bb 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -1,7 +1,5 @@ version: '3' name: zeppelin-prod -volumes: - mysql-data: {} services: nginx: build: @@ -24,7 +22,7 @@ services: ports: - ${DOCKER_PROD_MYSQL_PORT:?Missing DOCKER_PROD_MYSQL_PORT}:3306 volumes: - - mysql-data:/var/lib/mysql + - ./docker/production/data/mysql:/var/lib/mysql command: --authentication-policy=mysql_native_password prepare_backend: @@ -44,15 +42,18 @@ services: - prepare_backend volumes: - ./:/zeppelin - - # Wait for the build_backend container to finish before starting the bot - # See: https://github.com/docker/compose/issues/5007#issuecomment-335815508 user: node - command: |- - bash -c ' \ - while ping -c1 prepare_backend &>/dev/null; do sleep 1; done; \ - cd /zeppelin/backend && npm run start-api-prod \ - ' + command: ["/bin/bash", "/zeppelin/docker/production/start-api.sh"] + + bot: + image: node:16.16 + restart: on-failure + depends_on: + - prepare_backend + volumes: + - ./:/zeppelin + user: node + command: ["/bin/bash", "/zeppelin/docker/production/start-bot.sh"] build_dashboard: image: node:16.16 diff --git a/docker/development/data/mysql/.gitignore b/docker/development/data/mysql/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/docker/development/data/mysql/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docker/production/config/mysql.conf.d/.gitignore b/docker/production/config/mysql.conf.d/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/docker/production/config/mysql.conf.d/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docker/production/data/mysql/.gitignore b/docker/production/data/mysql/.gitignore new file mode 100644 index 00000000..d6b7ef32 --- /dev/null +++ b/docker/production/data/mysql/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/docker/production/start-api.sh b/docker/production/start-api.sh new file mode 100755 index 00000000..00e9359a --- /dev/null +++ b/docker/production/start-api.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# This wrapper script is used for two purposes: +# 1. Waiting for the prepare_backend container to finish before starting (see https://github.com/docker/compose/issues/5007#issuecomment-335815508) +# 2. Forwarding signals to the app (see https://unix.stackexchange.com/a/196053) + +# Wait for the backend preparations to finish before continuing +echo "Waiting for prepare_backend to finish before starting the API..." +while ping -c1 prepare_backend &>/dev/null; do sleep 1; done; + +echo "Starting the API" +cd /zeppelin/backend +exec npm run start-api-prod diff --git a/docker/production/start-bot.sh b/docker/production/start-bot.sh new file mode 100644 index 00000000..d9214465 --- /dev/null +++ b/docker/production/start-bot.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# This wrapper script is used for two purposes: +# 1. Waiting for the prepare_backend container to finish before starting (see https://github.com/docker/compose/issues/5007#issuecomment-335815508) +# 2. Forwarding signals to the app (see https://unix.stackexchange.com/a/196053) + +# Wait for the backend preparations to finish before continuing +echo "Waiting for prepare_backend to finish before starting the bot..." +while ping -c1 prepare_backend &>/dev/null; do sleep 1; done; + +echo "Starting the bot" +cd /zeppelin/backend +exec npm run start-bot-prod diff --git a/update.sh b/update.sh index bf615aa5..8a32cf94 100755 --- a/update.sh +++ b/update.sh @@ -1,4 +1,9 @@ #!/bin/bash -. ./update-backend.sh -. ./update-dashboard.sh +echo Updating Zeppelin... + +docker-compose -f docker-compose.production.yml down +git pull +docker-compose -f docker-compose.production.yml -d up + +echo Update finished! From a9cf019d2943ea84fe64bf8b120e7d76fc2cb51f Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sat, 6 Aug 2022 22:28:26 +0300 Subject: [PATCH 21/29] chore: add LICENCE.md --- LICENCE.md | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 LICENCE.md diff --git a/LICENCE.md b/LICENCE.md new file mode 100644 index 00000000..3d910e33 --- /dev/null +++ b/LICENCE.md @@ -0,0 +1,55 @@ +# Elastic License 2.0 (ELv2) + +## Elastic License + +### Acceptance + +By using the software, you agree to all of the terms and conditions below. + +### Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, non-sublicensable, non-transferable license to use, copy, distribute, make available, and prepare derivative works of the software, in each case subject to the limitations and conditions below. + +### Limitations + +You may not provide the software to third parties as a hosted or managed service, where the service provides users with access to any substantial set of the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality in the software, and you may not remove or obscure any functionality in the software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices of the licensor in the software. Any use of the licensor’s trademarks is subject to applicable law. + +### Patents + +The licensor grants you a license, under any patent claims the licensor can license, or becomes able to license, to make, have made, use, sell, offer for sale, import and have imported the software, in each case subject to the limitations and conditions in this license. This license does not cover any patent claims that you cause to be infringed by modifications or additions to the software. If you or your company make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company. + +### Notices + +You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the software prominent notices stating that you have modified the software. + +### No Other Rights + +These terms do not imply any licenses other than those expressly granted in these terms. + +### Termination + +If you use the software in violation of these terms, such use is not licensed, and your licenses will automatically terminate. If the licensor provides you with a notice of your violation, and you cease all violation of this license no later than 30 days after you receive that notice, your licenses will be reinstated retroactively. However, if you violate these terms after such reinstatement, any additional violation of these terms will cause your licenses to terminate automatically and permanently. + +### No Liability + +***As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.*** + +### Definitions + +The **licensor** is the entity offering these terms, and the **software** is the software the licensor makes available under these terms, including any portion of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. **control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. From ad1f5b8fde75db3527c71223506eeee9b175bb59 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sat, 6 Aug 2022 22:31:10 +0300 Subject: [PATCH 22/29] chore: rename LICENCE.md to LICENSE.md --- LICENCE.md => LICENSE.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename LICENCE.md => LICENSE.md (100%) diff --git a/LICENCE.md b/LICENSE.md similarity index 100% rename from LICENCE.md rename to LICENSE.md From 22d0b14ef16ea3d95095c9300cd5992b294bbf00 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 7 Aug 2022 12:23:27 +0300 Subject: [PATCH 23/29] docker: allow configuring UID/GID used inside the containers --- .env.example | 5 +++++ docker-compose.production.yml | 28 ++++++++++++++++++++-------- docker/production/node/Dockerfile | 10 ++++++++++ 3 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 docker/production/node/Dockerfile diff --git a/.env.example b/.env.example index 4494f3b0..363e617f 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,11 @@ API_PORT=3000 # Only required if relevant feature is used #PHISHERMAN_API_KEY= +# The user ID and group ID that should be used within the Docker containers +# This should match your own user ID and group ID. Run `id -u` and `id -g` to find them. +DOCKER_USER_UID= +DOCKER_USER_GID= + # # DOCKER (DEVELOPMENT) # diff --git a/docker-compose.production.yml b/docker-compose.production.yml index 109cb3bb..6ab79342 100644 --- a/docker-compose.production.yml +++ b/docker-compose.production.yml @@ -26,39 +26,51 @@ services: command: --authentication-policy=mysql_native_password prepare_backend: - image: node:16.16 + build: + context: ./docker/production/node + args: + DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID} + DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID} depends_on: - mysql volumes: - ./:/zeppelin - user: node command: |- bash -c "cd /zeppelin/backend && npm ci && npm run build && npm run migrate-prod" api: - image: node:16.16 + build: + context: ./docker/production/node + args: + DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID} + DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID} restart: on-failure depends_on: - prepare_backend volumes: - ./:/zeppelin - user: node command: ["/bin/bash", "/zeppelin/docker/production/start-api.sh"] bot: - image: node:16.16 + build: + context: ./docker/production/node + args: + DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID} + DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID} restart: on-failure depends_on: - prepare_backend volumes: - ./:/zeppelin - user: node command: ["/bin/bash", "/zeppelin/docker/production/start-bot.sh"] build_dashboard: - image: node:16.16 + build: + context: ./docker/production/node + args: + DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID} + DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID} volumes: - ./:/zeppelin - user: node command: |- bash -c "cd /zeppelin/dashboard && npm ci && npm run build" diff --git a/docker/production/node/Dockerfile b/docker/production/node/Dockerfile new file mode 100644 index 00000000..22b46e45 --- /dev/null +++ b/docker/production/node/Dockerfile @@ -0,0 +1,10 @@ +FROM node:16.16 + +ARG DOCKER_USER_UID +ARG DOCKER_USER_GID + +# This custom Dockerfile is needed for the Node image so we can change the uid/gid used for the node user +# See https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md#non-root-user +RUN groupmod -g "${DOCKER_USER_GID}" node && usermod -u "${DOCKER_USER_UID}" -g "${DOCKER_USER_GID}" node + +USER node From 9bd1b97a11b1f0e4bfd4ccd1a70d406538c5676c Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 7 Aug 2022 12:45:52 +0300 Subject: [PATCH 24/29] fix: ignore empty values for STAFF/DEFAULT_ALLOWED_SERVERS --- backend/src/env.ts | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/backend/src/env.ts b/backend/src/env.ts index 5a607e45..a125a744 100644 --- a/backend/src/env.ts +++ b/backend/src/env.ts @@ -15,9 +15,27 @@ const envType = z.object({ API_URL: z.string().url(), API_PORT: z.preprocess((v) => Number(v), z.number().min(1).max(65535)).default(3000), - STAFF: z.preprocess((v) => String(v).split(","), z.array(z.string())).optional(), + STAFF: z + .preprocess( + (v) => + String(v) + .split(",") + .map((s) => s.trim()) + .filter((s) => s !== ""), + z.array(z.string()), + ) + .optional(), - DEFAULT_ALLOWED_SERVERS: z.preprocess((v) => String(v).split(","), z.array(z.string())).optional(), + DEFAULT_ALLOWED_SERVERS: z + .preprocess( + (v) => + String(v) + .split(",") + .map((s) => s.trim()) + .filter((s) => s !== ""), + z.array(z.string()), + ) + .optional(), PHISHERMAN_API_KEY: z.string().optional(), From 561f91949d3e5255c0f70ee18131a36fc9ca5e2c Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 7 Aug 2022 13:52:04 +0300 Subject: [PATCH 25/29] docs: fixes/tweaks to docker docs --- DEVELOPMENT.md | 6 +++--- PRODUCTION.md | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 90f2c2ca..62e3b66a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -17,7 +17,7 @@ Below you can find instructions for setting up the environment and getting start 1. Install Docker 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` -4. Run `docker-compose up` to start the development environment +4. Run `docker compose -f docker-compose.development.yml up` to start the development environment 5. In VSCode: Install the `Remote - SSH` plugin 6. In VSCode: Run `Remote-SSH: Connect to Host...` * As the address, use `ubuntu@127.0.0.1:3002` (where `3002` matches `DOCKER_DEV_SSH_PORT` in `.env`) @@ -28,7 +28,7 @@ Below you can find instructions for setting up the environment and getting start 1. Install Docker 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` -4. Run `docker-compose up` to start the development environment +4. Run `docker compose -f docker-compose.development.yml up` to start the development environment 5. Choose `Connect via SSH` and create a new connection: * Username: `ubuntu` * Host: `127.0.0.1` @@ -43,7 +43,7 @@ Below you can find instructions for setting up the environment and getting start 1. Install Docker 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` -4. Run `docker-compose up` to start the development environment +4. Run `docker compose -f docker-compose.development.yml up` to start the development environment 5. Use the following credentials for connecting with your IDE: * Host: `127.0.0.1` * Port: `3002` (matching the `DOCKER_DEV_SSH_PORT` value in `.env`) diff --git a/PRODUCTION.md b/PRODUCTION.md index a45bf50d..2a9b1baa 100644 --- a/PRODUCTION.md +++ b/PRODUCTION.md @@ -5,7 +5,7 @@ Zeppelin's production environment - that is, the **bot, API, and dashboard** - u 1. Install Docker on the machine running the bot 2. Make a copy of `.env.example` called `.env` 3. Fill in the missing values in `.env` -4. Run `docker-compose -f docker-compose.production.yml -d up` +4. Run `docker compose -f docker-compose.production.yml up -d` ## Updating the bot @@ -13,15 +13,15 @@ Zeppelin's production environment - that is, the **bot, API, and dashboard** - u If you've downloaded the bot's files by cloning the git repository, you can use `update.sh` to update the bot. ### Manual instructions -1. Shut the bot down: `docker-compose -f docker-compose.production.yml down` +1. Shut the bot down: `docker compose -f docker-compose.production.yml stop` 2. Update the files (e.g. `git pull`) -3. Start the bot again: `docker-compose -f docker-compose.production.yml -d up` +3. Start the bot again: `docker compose -f docker-compose.production.yml start` ### Ephemeral hotfixes If you need to make a hotfix to the bot's source files directly on the server: -1. Shut the bot down: `docker-compose -f docker-compose.production.yml down` +1. Shut the bot down: `docker compose -f docker-compose.production.yml stop` 2. Make your edits -3. Start the bot again: `docker-compose -f docker-compose.production.yml -d up` +3. Start the bot again: `docker compose -f docker-compose.production.yml start` Note that you can't edit the compiled files directly as they're overwritten when the environment starts. Only edit files in `/backend/src`, `/shared/src`, and `/dashboard/src`. @@ -29,4 +29,4 @@ Only edit files in `/backend/src`, `/shared/src`, and `/dashboard/src`. Make sure to revert any hotfixes before updating the bot normally. ## View logs -To view real-time logs, run `docker-compose -f docker-compose.production.yml -t logs` +To view real-time logs, run `docker compose -f docker-compose.production.yml -t -f logs` From 1308054d6903b433dc4d4cbcd1d49d28e10099c9 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 7 Aug 2022 13:52:41 +0300 Subject: [PATCH 26/29] fix: use docker compose v2 syntax in update.sh; only stop/start the containers rather than recreating them --- update.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/update.sh b/update.sh index 8a32cf94..870e2e2e 100755 --- a/update.sh +++ b/update.sh @@ -2,8 +2,8 @@ echo Updating Zeppelin... -docker-compose -f docker-compose.production.yml down +docker compose -f docker-compose.production.yml stop git pull -docker-compose -f docker-compose.production.yml -d up +docker compose -f docker-compose.production.yml start echo Update finished! From 030da51da1c95f143f180eb1c6f3d42e7e780d53 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 7 Aug 2022 13:53:02 +0300 Subject: [PATCH 27/29] chore: remove deprecated .env.example from /dashboard --- dashboard/.env.example | 1 - 1 file changed, 1 deletion(-) delete mode 100644 dashboard/.env.example diff --git a/dashboard/.env.example b/dashboard/.env.example deleted file mode 100644 index fb6be76e..00000000 --- a/dashboard/.env.example +++ /dev/null @@ -1 +0,0 @@ -API_URL= From c01a4027bda0a2e528b1b3ac9f5a124ca83190d1 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 7 Aug 2022 13:55:07 +0300 Subject: [PATCH 28/29] chore: remove files that are no longer needed with the docker environment --- process-api.json | 12 ------------ process-bot.json | 12 ------------ update-backend-hotfix.sh | 16 ---------------- update-backend.sh | 25 ------------------------- update-dashboard.sh | 18 ------------------ 5 files changed, 83 deletions(-) delete mode 100644 process-api.json delete mode 100644 process-bot.json delete mode 100755 update-backend-hotfix.sh delete mode 100755 update-backend.sh delete mode 100755 update-dashboard.sh diff --git a/process-api.json b/process-api.json deleted file mode 100644 index 2feaf75f..00000000 --- a/process-api.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "apps": [ - { - "name": "zeppelin-api", - "cwd": "./backend", - "script": "npm", - "args": "run start-api-prod", - "log_date_format": "YYYY-MM-DD HH:mm:ss.SSS", - "exp_backoff_restart_delay": 2500 - } - ] -} diff --git a/process-bot.json b/process-bot.json deleted file mode 100644 index b8667e2d..00000000 --- a/process-bot.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "apps": [ - { - "name": "zeppelin", - "cwd": "./backend", - "script": "npm", - "args": "run start-bot-prod", - "log_date_format": "YYYY-MM-DD HH:mm:ss.SSS", - "exp_backoff_restart_delay": 2500 - } - ] -} diff --git a/update-backend-hotfix.sh b/update-backend-hotfix.sh deleted file mode 100755 index e692d7a9..00000000 --- a/update-backend-hotfix.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Load nvm -. ~/.nvm/nvm.sh - -# Run hotfix update -cd backend -nvm use -git pull -npm run build - -# Restart processes -cd .. -nvm use -pm2 restart process-bot.json -pm2 restart process-api.json diff --git a/update-backend.sh b/update-backend.sh deleted file mode 100755 index 4da51186..00000000 --- a/update-backend.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# Load nvm -. ~/.nvm/nvm.sh - -# Stop current processes -nvm use -pm2 delete process-bot.json -pm2 delete process-api.json - -# Run update -nvm use -git pull -npm ci - -cd backend -npm ci -npm run build -npm run migrate-prod - -# Start processes again -cd .. -nvm use -pm2 start process-bot.json -pm2 start process-api.json diff --git a/update-dashboard.sh b/update-dashboard.sh deleted file mode 100755 index 34a47fc6..00000000 --- a/update-dashboard.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -TARGET_DIR=/var/www/zeppelin.gg - -# Load nvm -. ~/.nvm/nvm.sh - -# Update dashboard -cd dashboard -git pull -nvm use -npm ci -npm run build -rm -r $TARGET_DIR/* -cp -R dist/* $TARGET_DIR - -# Return -cd .. From 7d47f36d991e539d83c4ddc2a59c095451731a00 Mon Sep 17 00:00:00 2001 From: Dragory <2606411+Dragory@users.noreply.github.com> Date: Sun, 7 Aug 2022 13:58:47 +0300 Subject: [PATCH 29/29] docs: documentation updates --- .env.example | 4 ++++ PRODUCTION.md | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.env.example b/.env.example index 363e617f..aadba71b 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,8 @@ CLIENT_ID= CLIENT_SECRET= BOT_TOKEN= +# The defaults here automatically work for the development environment. +# For production, change localhost:3300 to your domain. DASHBOARD_URL=https://localhost:3300 API_URL=https://localhost:3300/api @@ -28,6 +30,7 @@ DOCKER_USER_GID= # # DOCKER (DEVELOPMENT) +# NOTE: You only need to fill in these values for running the development environment. See production config further below. # DOCKER_DEV_WEB_PORT=3300 @@ -50,6 +53,7 @@ DOCKER_DEV_SSH_PASSWORD=password # # DOCKER (PRODUCTION) +# NOTE: You only need to fill in these values for running the production environment. See development config above. # DOCKER_PROD_DOMAIN= diff --git a/PRODUCTION.md b/PRODUCTION.md index 2a9b1baa..e05d5cc3 100644 --- a/PRODUCTION.md +++ b/PRODUCTION.md @@ -7,6 +7,8 @@ Zeppelin's production environment - that is, the **bot, API, and dashboard** - u 3. Fill in the missing values in `.env` 4. Run `docker compose -f docker-compose.production.yml up -d` +**Note:** The dashboard and API are exposed with a self-signed certificate. It is recommended to set up a proxy with a proper certificate in front of them. Cloudflare is a popular choice here. + ## Updating the bot ### One-click script