Merge branch '220601_docker_wip'

.devcontainer/devcontainer.json

@@ -0,0 +1,9 @@
+{
+  "name": "Zeppelin Development",
+
+  "dockerComposeFile": "../docker-compose.development.yml",
+
+  "service": "devenv",
+  "remoteUser": "ubuntu",
+  "workspaceFolder": "/home/ubuntu/zeppelin"
+}

.env.example

@@ -1 +1,74 @@
-KEY=32_character_encryption_key
+# 32 character encryption key
+KEY=
+
+# Values from the Discord developer portal
+CLIENT_ID=
+CLIENT_SECRET=
+BOT_TOKEN=
+
+# The defaults here automatically work for the development environment.
+# For production, change localhost:3300 to your domain.
+DASHBOARD_URL=https://localhost:3300
+API_URL=https://localhost:3300/api
+
+# Comma-separated list of user IDs who should have access to the bot's global commands
+STAFF=
+
+# A comma-separated list of server IDs that should be allowed by default
+DEFAULT_ALLOWED_SERVERS=
+
+# When using the Docker-based development environment, this is only used internally. The API will be available at localhost:DOCKER_DEV_WEB_PORT/api.
+API_PORT=3000
+
+# Only required if relevant feature is used
+#PHISHERMAN_API_KEY=
+
+# The user ID and group ID that should be used within the Docker containers
+# This should match your own user ID and group ID. Run `id -u` and `id -g` to find them.
+DOCKER_USER_UID=
+DOCKER_USER_GID=
+
+#
+# DOCKER (DEVELOPMENT)
+# NOTE: You only need to fill in these values for running the development environment. See production config further below.
+#
+
+DOCKER_DEV_WEB_PORT=3300
+
+# The MySQL database running in the container is exposed to the host on this port,
+# allowing access with database tools such as DBeaver
+DOCKER_DEV_MYSQL_PORT=3001
+# Password for the Zeppelin database user
+DOCKER_DEV_MYSQL_PASSWORD=
+# Password for the MySQL root user
+DOCKER_DEV_MYSQL_ROOT_PASSWORD=
+
+# The development environment container has an SSH server that you can connect to.
+# This is the port that server is exposed to the host on.
+DOCKER_DEV_SSH_PORT=3002
+DOCKER_DEV_SSH_PASSWORD=password
+
+# If your user has a different UID than 1000, you might have to fill that in here to avoid permission issues
+#DOCKER_DEV_UID=1000
+
+#
+# DOCKER (PRODUCTION)
+# NOTE: You only need to fill in these values for running the production environment. See development config above.
+#
+
+DOCKER_PROD_DOMAIN=
+DOCKER_PROD_WEB_PORT=443
+# The MySQL database running in the container is exposed to the host on this port,
+# allowing access with database tools such as DBeaver
+DOCKER_PROD_MYSQL_PORT=3001
+# Password for the Zeppelin database user
+DOCKER_PROD_MYSQL_PASSWORD=
+# Password for the MySQL root user
+DOCKER_PROD_MYSQL_ROOT_PASSWORD=
+
+# You only need to set these if you're running an external database.
+# In a standard setup, the database is run in a docker container.
+#DB_HOST=
+#DB_USER=
+#DB_PASSWORD=
+#DB_DATABASE=

DEVELOPMENT.md

@@ -0,0 +1,71 @@
+# Zeppelin development environment
+Zeppelin's development environment runs entirely within a Docker container.
+Below you can find instructions for setting up the environment and getting started with development!
+
+**Note:** If you'd just like to run the bot for your own server, see 👉 **[PRODUCTION.md](./PRODUCTION.md)** 👈
+
+## Starting the development environment
+
+### Using VSCode devcontainers
+1. Install Docker
+2. Make a copy of `.env.example` called `.env`
+3. Fill in the missing values in `.env`
+4. In VSCode: Install the `Remote - Containers` plugin
+5. In VSCode: Run `Remote-Containers: Open Folder in Container...` and select the Zeppelin folder
+
+### Using VSCode remote SSH plugin
+1. Install Docker
+2. Make a copy of `.env.example` called `.env`
+3. Fill in the missing values in `.env`
+4. Run `docker compose -f docker-compose.development.yml up` to start the development environment
+5. In VSCode: Install the `Remote - SSH` plugin
+6. In VSCode: Run `Remote-SSH: Connect to Host...`
+    * As the address, use `ubuntu@127.0.0.1:3002` (where `3002` matches `DOCKER_DEV_SSH_PORT` in `.env`)
+    * Use the password specified in `.env` as `DOCKER_DEV_SSH_PASSWORD`
+7. In VSCode: Once connected, click `Open folder...` and select `/home/ubuntu/zeppelin`
+
+### Using JetBrains Gateway
+1. Install Docker
+2. Make a copy of `.env.example` called `.env`
+3. Fill in the missing values in `.env`
+4. Run `docker compose -f docker-compose.development.yml up` to start the development environment
+5. Choose `Connect via SSH` and create a new connection:
+    * Username: `ubuntu`
+    * Host: `127.0.0.1`
+    * Port: `3002` (matching the `DOCKER_DEV_SSH_PORT` value in `.env`)
+6. Click `Check Connection and Continue` and enter the password specified in `.env` as `DOCKER_DEV_SSH_PASSWORD` when asked
+7. In the next pane:
+    * IDE version: WebStorm, PHPStorm, or IntelliJ IDEA
+    * Project directory: `/home/ubuntu/zeppelin`
+8. Click `Download and Start IDE`
+
+### Using any other IDE with SSH development support
+1. Install Docker
+2. Make a copy of `.env.example` called `.env`
+3. Fill in the missing values in `.env`
+4. Run `docker compose -f docker-compose.development.yml up` to start the development environment
+5. Use the following credentials for connecting with your IDE:
+    * Host: `127.0.0.1`
+    * Port: `3002` (matching the `DOCKER_DEV_SSH_PORT` value in `.env`)
+    * Username: `ubuntu`
+    * Password: As specified in `.env` as `DOCKER_DEV_SSH_PASSWORD`
+
+## Starting the project
+
+### Starting the backend (bot + api)
+These commands are run inside the dev container. You should be able to open a terminal in your IDE after connecting.
+
+1. `cd ~/zeppelin/backend`
+2. `npm ci`
+3. `npm run migrate-dev`
+4. `npm run watch`
+
+### Starting the dashboard
+These commands are run inside the dev container. You should be able to open a terminal in your IDE after connecting.
+
+1. `cd ~/zeppelin/dashboard`
+2. `npm ci`
+3. `npm run watch-build`
+
+### Opening the dashboard
+Browse to https://localhost:3300 to view the dashboard

LICENSE.md

@@ -0,0 +1,55 @@
+# Elastic License 2.0 (ELv2)
+
+## Elastic License
+
+### Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+### Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide, non-sublicensable, non-transferable license to use, copy, distribute, make available, and prepare derivative works of the software, in each case subject to the limitations and conditions below.
+
+### Limitations
+
+You may not provide the software to third parties as a hosted or managed service, where the service provides users with access to any substantial set of the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality in the software, and you may not remove or obscure any functionality in the software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices of the licensor in the software. Any use of the licensor’s trademarks is subject to applicable law.
+
+### Patents
+
+The licensor grants you a license, under any patent claims the licensor can license, or becomes able to license, to make, have made, use, sell, offer for sale, import and have imported the software, in each case subject to the limitations and conditions in this license. This license does not cover any patent claims that you cause to be infringed by modifications or additions to the software. If you or your company make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
+
+### Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the software prominent notices stating that you have modified the software.
+
+### No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in these terms.
+
+### Termination
+
+If you use the software in violation of these terms, such use is not licensed, and your licenses will automatically terminate. If the licensor provides you with a notice of your violation, and you cease all violation of this license no later than 30 days after you receive that notice, your licenses will be reinstated retroactively. However, if you violate these terms after such reinstatement, any additional violation of these terms will cause your licenses to terminate automatically and permanently.
+
+### No Liability
+
+***As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.***
+
+### Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the software the licensor makes available under these terms, including any portion of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. **control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

MANAGEMENT.md

@@ -0,0 +1,34 @@
+# Management
+After starting Zeppelin -- either in the [development](./DEVELOPMENT.md) or [production](./PRODUCTION.md) environment -- you have several tools available to manage it.
+
+## Note
+Make sure to add yourself to the list of staff members (`STAFF`) in `.env` and allow at least one server by default (`DEFAULT_ALLOWED_SERVERS`). Then, invite the bot to the server.
+
+In all examples below, `@Bot` refers to a user mention of the bot user. Make sure to run the commands on a server with the bot, in a channel that the bot can see.
+
+In the command parameters, `<this>` refers to a required parameter (don't include the `< >` symbols) and `[this]` refers to an optional parameter (don't include the `[ ]` symbols). `<this...>` refers to being able to list multiple values, e.g. `value1 value2 value3`.
+
+## Allow a server to invite the bot
+Run the following command:
+```
+@Bot allow_server <serverId> [userId]
+```
+When specifying a user ID, that user will be given "Bot manager" level access to the server's dashboard, allowing them to manage access for other users.
+
+## Disallow a server
+Run the following command:
+```
+@Bot disallow_server <serverId>
+```
+
+## Grant access to a server's dashboard
+Run the following command:
+```
+@Bot add_dashboard_user <serverId> <userId...>
+```
+
+## Remove access to a server's dashboard
+Run the following command:
+```
+@Bot remove_dashboard_user <serverId> <userId...>
+```

PRODUCTION.md

@@ -0,0 +1,34 @@
+# Zeppelin production environment
+Zeppelin's production environment - that is, the **bot, API, and dashboard** - uses Docker.
+
+## Starting the production environment
+1. Install Docker on the machine running the bot
+2. Make a copy of `.env.example` called `.env`
+3. Fill in the missing values in `.env`
+4. Run `docker compose -f docker-compose.production.yml up -d`
+
+**Note:** The dashboard and API are exposed with a self-signed certificate. It is recommended to set up a proxy with a proper certificate in front of them. Cloudflare is a popular choice here.
+
+## Updating the bot
+
+### One-click script
+If you've downloaded the bot's files by cloning the git repository, you can use `update.sh` to update the bot.
+
+### Manual instructions
+1. Shut the bot down: `docker compose -f docker-compose.production.yml stop`
+2. Update the files (e.g. `git pull`)
+3. Start the bot again: `docker compose -f docker-compose.production.yml start`
+
+### Ephemeral hotfixes
+If you need to make a hotfix to the bot's source files directly on the server:
+1. Shut the bot down: `docker compose -f docker-compose.production.yml stop`
+2. Make your edits
+3. Start the bot again: `docker compose -f docker-compose.production.yml start`
+
+Note that you can't edit the compiled files directly as they're overwritten when the environment starts.
+Only edit files in `/backend/src`, `/shared/src`, and `/dashboard/src`.
+
+Make sure to revert any hotfixes before updating the bot normally.
+
+## View logs
+To view real-time logs, run `docker compose -f docker-compose.production.yml -t -f logs`

README.md

@@ -19,89 +19,15 @@ Zeppelin is a moderation bot for Discord, designed with large servers and reliab
 
 See https://zeppelin.gg/ for more details.
 
+## Usage documentation
+For information on how to use the bot, see https://zeppelin.gg/docs
+
 ## Development
-These instructions are intended for bot development only.
+See [DEVELOPMENT.md](./DEVELOPMENT.md) for instructions on running the development environment.
 
-👉 **No support is offered for self-hosting the bot!** 👈
+Once you have the environment up and running, see [MANAGEMENT.md](./MANAGEMENT.md) for how to manage your bot.
 
-### Running the bot
-1. `cd backend`
-2. `npm ci`
-3. Make a copy of `bot.env.example` called `bot.env`, fill in the values
-4. Run the desired start script:
-    * `npm run build` followed by `npm run start-bot-dev` to run the bot in a **development** environment
-    * `npm run build` followed by `npm run start-bot-prod` to run the bot in a **production** environment
-    * `npm run watch` to watch files and run the **bot and api both** in a **development** environment
-      with automatic restart on file changes
-5. When testing, make sure you have your test server in the `allowed_guilds` table or the guild's config won't be loaded at all
+## Production
+See [PRODUCTION.md](./PRODUCTION.md) for instructions on how to run the bot in production.
 
-### Running the API server
-1. `cd backend`
-2. `npm ci`
-3. Make a copy of `api.env.example` called `api.env`, fill in the values
-4. Run the desired start script:
-    * `npm run build` followed by `npm run start-api-dev` to run the api in a **development** environment
-    * `npm run build` followed by `npm run start-api-prod` to run the api in a **production** environment
-    * `npm run watch` to watch files and run the **bot and api both** in a **development** environment
-      with automatic restart on file changes
-
-### Running the dashboard
-1. `cd dashboard`
-2. `npm ci`
-3. Make a copy of `.env.example` called `.env`, fill in the values
-4. Run the desired start script:
-    * `npm run build` compiles the dashboard's static files to `dist/` which can then be served with any web server
-    * `npm run watch` runs webpack's dev server that automatically reloads on changes
-
-### Notes
-* Since we now use shared paths in `tsconfig.json`, the compiled files in `backend/dist/` have longer paths, e.g.
-  `backend/dist/backend/src/index.js` instead of `backend/dist/index.js`. This is because the compiled shared files
-  are placed in `backend/dist/shared`.
-* The `backend/register-tsconfig-paths.js` module takes care of registering shared paths from `tsconfig.json` for
-  `ava` and compiled `.js` files
-* To run the tests for the files in the `shared/` directory, you also need to run `npm ci` there
-
-### Config format example
-Configuration is stored in the database in the `configs` table
-
-```yml
-prefix: '!'
-
-# role id: level
-levels:
-  "12345678": 100 # Example admin
-  "98765432": 50 # Example mod
-
-plugins:
-  mod_plugin:
-    config:
-      kick_message: 'You have been kicked'
-      can_kick: false
-    overrides:
-      - level: '>=50'
-        config:
-          can_kick: true
-      - level: '>=100'
-        config:
-          kick_message: 'You have been kicked by an admin'
-
-  other_plugin:
-    config:
-      categories:
-        mycategory:
-          opt: "something"
-        othercategory:
-          enabled: false
-          opt: "hello"
-    overrides:
-      - level: '>=50'
-        config:
-          categories:
-            mycategory:
-              enabled: false
-      - channel: '1234'
-        config:
-          categories:
-            othercategory:
-              enabled: true
-```
+Once you have the environment up and running, see [MANAGEMENT.md](./MANAGEMENT.md) for how to manage your bot.

backend/api.env.example

@@ -1,10 +0,0 @@
-PORT=
-CLIENT_ID=
-CLIENT_SECRET=
-OAUTH_CALLBACK_URL=
-DASHBOARD_URL=
-DB_HOST=
-DB_USER=
-DB_PASSWORD=
-DB_DATABASE=
-STAFF=

backend/bot.env.example

@@ -1,7 +0,0 @@
-TOKEN=
-DB_HOST=
-DB_USER=
-DB_PASSWORD=
-DB_DATABASE=
-PROFILING=false
-PHISHERMAN_API_KEY=

backend/ormconfig.js

@@ -1,38 +1,23 @@
 const fs = require("fs");
 const path = require("path");
 const pkgUp = require("pkg-up");
-
-const closestPackageJson = pkgUp.sync();
-if (!closestPackageJson) {
-  throw new Error("Could not find project root from ormconfig.js");
-}
-const backendRoot = path.dirname(closestPackageJson);
-
-try {
-  fs.accessSync(path.resolve(backendRoot, "bot.env"));
-  require("dotenv").config({ path: path.resolve(backendRoot, "bot.env") });
-} catch {
-  try {
-    fs.accessSync(path.resolve(backendRoot, "api.env"));
-    require("dotenv").config({ path: path.resolve(backendRoot, "api.env") });
-  } catch {
-    throw new Error("bot.env or api.env required");
-  }
-}
+const { backendDir } = require("./dist/backend/src/paths");
+const { env } = require("./dist/backend/src/env");
 
 const moment = require("moment-timezone");
 moment.tz.setDefault("UTC");
 
-const entities = path.relative(process.cwd(), path.resolve(backendRoot, "dist/backend/src/data/entities/*.js"));
-const migrations = path.relative(process.cwd(), path.resolve(backendRoot, "dist/backend/src/migrations/*.js"));
-const migrationsDir = path.relative(process.cwd(), path.resolve(backendRoot, "src/migrations"));
+const entities = path.relative(process.cwd(), path.resolve(backendDir, "dist/backend/src/data/entities/*.js"));
+const migrations = path.relative(process.cwd(), path.resolve(backendDir, "dist/backend/src/migrations/*.js"));
+const migrationsDir = path.relative(process.cwd(), path.resolve(backendDir, "src/migrations"));
 
 module.exports = {
   type: "mysql",
-  host: process.env.DB_HOST,
-  username: process.env.DB_USER,
-  password: process.env.DB_PASSWORD,
-  database: process.env.DB_DATABASE,
+  host: env.DB_HOST,
+  port: env.DB_PORT,
+  username: env.DB_USER,
+  password: env.DB_PASSWORD,
+  database: env.DB_DATABASE,
   charset: "utf8mb4",
   supportBigNumbers: true,
   bigNumberStrings: true,

backend/package.json

@@ -14,9 +14,9 @@
     "start-api-prod": "cross-env NODE_ENV=production node -r ./register-tsconfig-paths.js --unhandled-rejections=strict --enable-source-maps --stack-trace-limit=30 dist/backend/src/api/index.js",
     "watch-api": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-api-dev\"",
     "typeorm": "node -r ./register-tsconfig-paths.js ./node_modules/typeorm/cli.js",
-    "migrate-prod": "npm run typeorm -- migration:run",
+    "migrate-prod": "cross-env NODE_ENV=production npm run typeorm -- migration:run",
     "migrate-dev": "npm run build && npm run typeorm -- migration:run",
-    "migrate-rollback-prod": "npm run typeorm -- migration:revert",
+    "migrate-rollback-prod": "cross-env NODE_ENV=production npm run typeorm -- migration:revert",
     "migrate-rollback-dev": "npm run build && npm run typeorm -- migration:revert",
     "test": "npm run build && npm run run-tests",
     "run-tests": "ava",

backend/src/api/auth.ts

@@ -9,6 +9,7 @@ import { ApiPermissionAssignments } from "../data/ApiPermissionAssignments";
 import { ApiUserInfo } from "../data/ApiUserInfo";
 import { ApiUserInfoData } from "../data/entities/ApiUserInfo";
 import { ok } from "./responses";
+import { env } from "../env";
 
 interface IPassportApiUser {
   apiKey: string;
@@ -54,22 +55,6 @@ function simpleDiscordAPIRequest(bearerToken, path): Promise<any> {
 export function initAuth(app: express.Express) {
   app.use(passport.initialize());
 
-  if (!process.env.CLIENT_ID) {
-    throw new Error("Auth: CLIENT ID missing");
-  }
-
-  if (!process.env.CLIENT_SECRET) {
-    throw new Error("Auth: CLIENT SECRET missing");
-  }
-
-  if (!process.env.OAUTH_CALLBACK_URL) {
-    throw new Error("Auth: OAUTH CALLBACK URL missing");
-  }
-
-  if (!process.env.DASHBOARD_URL) {
-    throw new Error("DASHBOARD_URL missing!");
-  }
-
   passport.serializeUser((user, done) => done(null, user));
   passport.deserializeUser((user, done) => done(null, user));
 
@@ -101,9 +86,9 @@ export function initAuth(app: express.Express) {
       {
         authorizationURL: "https://discord.com/api/oauth2/authorize",
         tokenURL: "https://discord.com/api/oauth2/token",
-        clientID: process.env.CLIENT_ID,
-        clientSecret: process.env.CLIENT_SECRET,
-        callbackURL: process.env.OAUTH_CALLBACK_URL,
+        clientID: env.CLIENT_ID,
+        clientSecret: env.CLIENT_SECRET,
+        callbackURL: `${env.API_URL}/auth/oauth-callback`,
         scope: ["identify"],
       },
       async (accessToken, refreshToken, profile, cb) => {
@@ -132,9 +117,9 @@ export function initAuth(app: express.Express) {
     passport.authenticate("oauth2", { failureRedirect: "/", session: false }),
     (req: Request, res: Response) => {
       if (req.user && req.user.apiKey) {
-        res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?apiKey=${req.user.apiKey}`);
+        res.redirect(`${env.DASHBOARD_URL}/login-callback/?apiKey=${req.user.apiKey}`);
       } else {
-        res.redirect(`${process.env.DASHBOARD_URL}/login-callback/?error=noAccess`);
+        res.redirect(`${env.DASHBOARD_URL}/login-callback/?error=noAccess`);
       }
     },
   );

backend/src/api/index.ts

@@ -1,8 +1,8 @@
 import { connect } from "../data/db";
 import { setIsAPI } from "../globals";
-import "./loadEnv";
+import { env } from "../env";
 
-if (!process.env.KEY) {
+if (!env.KEY) {
   // tslint:disable-next-line:no-console
   console.error("Project root .env with KEY is required!");
   process.exit(1);

backend/src/api/loadEnv.ts

@@ -1,4 +0,0 @@
-import path from "path";
-
-require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") });
-require("dotenv").config({ path: path.resolve(process.cwd(), "api.env") });

backend/src/api/start.ts

@@ -8,12 +8,13 @@ import { initGuildsAPI } from "./guilds/index";
 import { clientError, error, notFound } from "./responses";
 import { startBackgroundTasks } from "./tasks";
 import multer from "multer";
+import { env } from "../env";
 
 const app = express();
 
 app.use(
   cors({
-    origin: process.env.DASHBOARD_URL,
+    origin: env.DASHBOARD_URL,
   }),
 );
 app.use(
@@ -48,7 +49,7 @@ app.use((req, res, next) => {
   return notFound(res);
 });
 
-const port = (process.env.PORT && parseInt(process.env.PORT, 10)) || 3000;
+const port = env.API_PORT;
 app.listen(port, "0.0.0.0", () => console.log(`API server listening on port ${port}`)); // tslint:disable-line
 
 startBackgroundTasks();

backend/src/data/AllowedGuilds.ts

@@ -4,6 +4,7 @@ import { BaseRepository } from "./BaseRepository";
 import { AllowedGuild } from "./entities/AllowedGuild";
 import moment from "moment-timezone";
 import { DBDateFormat } from "../utils";
+import { env } from "../env";
 
 export class AllowedGuilds extends BaseRepository {
   private allowedGuilds: Repository<AllowedGuild>;

backend/src/data/Phisherman.ts

@@ -6,9 +6,10 @@ import { DAYS, DBDateFormat, HOURS, MINUTES } from "../utils";
 import moment from "moment-timezone";
 import { PhishermanKeyCacheEntry } from "./entities/PhishermanKeyCacheEntry";
 import crypto from "crypto";
+import { env } from "../env";
 
 const API_URL = "https://api.phisherman.gg";
-const MASTER_API_KEY = process.env.PHISHERMAN_API_KEY;
+const MASTER_API_KEY = env.PHISHERMAN_API_KEY;
 
 let caughtDomainTrackingMap: Map<string, Map<string, number[]>> = new Map();
 

backend/src/data/cleanup/configs.ts

@@ -9,6 +9,9 @@ const CLEAN_PER_LOOP = 50;
 export async function cleanupConfigs() {
   const configRepository = getRepository(Config);
 
+  // FIXME: The query below doesn't work on MySQL 8.0. Pending an update.
+  return;
+
   let cleaned = 0;
   let rows;
 

backend/src/data/db.ts

@@ -1,7 +1,11 @@
 import { Connection, createConnection } from "typeorm";
 import { SimpleError } from "../SimpleError";
-import connectionOptions from "../../ormconfig";
 import { QueryLogger } from "./queryLogger";
+import path from "path";
+import { backendDir } from "../paths";
+
+const ormconfigPath = path.join(backendDir, "ormconfig.js");
+const connectionOptions = require(ormconfigPath);
 
 let connectionPromise: Promise<Connection>;
 

backend/src/env.ts

@@ -0,0 +1,70 @@
+import path from "path";
+import fs from "fs";
+import dotenv from "dotenv";
+import { rootDir } from "./paths";
+import { z } from "zod";
+
+const envType = z.object({
+  KEY: z.string().length(32),
+
+  CLIENT_ID: z.string().min(16),
+  CLIENT_SECRET: z.string().length(32),
+  BOT_TOKEN: z.string().min(50),
+
+  DASHBOARD_URL: z.string().url(),
+  API_URL: z.string().url(),
+  API_PORT: z.preprocess((v) => Number(v), z.number().min(1).max(65535)).default(3000),
+
+  STAFF: z
+    .preprocess(
+      (v) =>
+        String(v)
+          .split(",")
+          .map((s) => s.trim())
+          .filter((s) => s !== ""),
+      z.array(z.string()),
+    )
+    .optional(),
+
+  DEFAULT_ALLOWED_SERVERS: z
+    .preprocess(
+      (v) =>
+        String(v)
+          .split(",")
+          .map((s) => s.trim())
+          .filter((s) => s !== ""),
+      z.array(z.string()),
+    )
+    .optional(),
+
+  PHISHERMAN_API_KEY: z.string().optional(),
+
+  DOCKER_DEV_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in development
+  DOCKER_PROD_MYSQL_PASSWORD: z.string().optional(), // Included here for the DB_PASSWORD default in production
+
+  DB_HOST: z.string().optional().default("mysql"),
+  DB_PORT: z
+    .preprocess((v) => Number(v), z.number())
+    .optional()
+    .default(3306),
+  DB_USER: z.string().optional().default("zeppelin"),
+  DB_PASSWORD: z.string().optional(), // Default is set to DOCKER_MYSQL_PASSWORD further below
+  DB_DATABASE: z.string().optional().default("zeppelin"),
+});
+
+let toValidate = {};
+const envPath = path.join(rootDir, ".env");
+if (fs.existsSync(envPath)) {
+  const buf = fs.readFileSync(envPath);
+  toValidate = dotenv.parse(buf);
+}
+
+export const env = envType.parse(toValidate);
+
+if (!env.DB_PASSWORD) {
+  if (process.env.NODE_ENV === "production" && env.DOCKER_PROD_MYSQL_PASSWORD) {
+    env.DB_PASSWORD = env.DOCKER_PROD_MYSQL_PASSWORD;
+  } else if (env.DOCKER_DEV_MYSQL_PASSWORD) {
+    env.DB_PASSWORD = env.DOCKER_DEV_MYSQL_PASSWORD;
+  }
+}

backend/src/index.ts

@@ -10,7 +10,6 @@ import { connect } from "./data/db";
 import { GuildLogs } from "./data/GuildLogs";
 import { LogType } from "./data/LogType";
 import { DiscordJSError } from "./DiscordJSError";
-import "./loadEnv";
 import { logger } from "./logger";
 import { baseGuildPlugins, globalPlugins, guildPlugins } from "./plugins/availablePlugins";
 import { RecoverablePluginError } from "./RecoverablePluginError";
@@ -37,12 +36,7 @@ import { runPhishermanCacheCleanupLoop, runPhishermanReportingLoop } from "./dat
 import { hasPhishermanMasterAPIKey } from "./data/Phisherman";
 import { consumeQueryStats } from "./data/queryLogger";
 import { EventEmitter } from "events";
-
-if (!process.env.KEY) {
-  // tslint:disable-next-line:no-console
-  console.error("Project root .env with KEY is required!");
-  process.exit(1);
-}
+import { env } from "./env";
 
 // Error handling
 let recentPluginErrors = 0;
@@ -413,5 +407,5 @@ connect().then(async () => {
   bot.initialize();
   logger.info("Bot Initialized");
   logger.info("Logging in...");
-  await client.login(process.env.TOKEN);
+  await client.login(env.BOT_TOKEN);
 });

backend/src/loadEnv.ts

@@ -1,4 +0,0 @@
-import path from "path";
-
-require("dotenv").config({ path: path.resolve(process.cwd(), "../.env") });
-require("dotenv").config({ path: path.resolve(process.cwd(), "bot.env") });

backend/src/pluginUtils.ts

@@ -14,6 +14,7 @@ import { TZeppelinKnub } from "./types";
 import { deepKeyIntersect, errorMessage, successMessage, tDeepPartial, tNullable } from "./utils";
 import { Tail } from "./utils/typeUtils";
 import { decodeAndValidateStrict, StrictValidationError, validate } from "./validatorUtils";
+import { isStaff } from "./staff";
 
 const { getMemberLevel } = helpers;
 
@@ -242,8 +243,8 @@ export function isOwner(pluginData: AnyPluginData<any>, userId: string) {
   return owners.includes(userId);
 }
 
-export const isOwnerPreFilter = (_, context: CommandContext<any>) => {
-  return isOwner(context.pluginData, context.message.author.id);
+export const isStaffPreFilter = (_, context: CommandContext<any>) => {
+  return isStaff(context.message.author.id);
 };
 
 type AnyFn = (...args: any[]) => any;

backend/src/plugins/Automod/actions/clean.ts

@@ -4,8 +4,6 @@ import { LogType } from "../../../data/LogType";
 import { noop } from "../../../utils";
 import { automodAction } from "../helpers";
 
-const cleanDebugServer = process.env.TEMP_CLEAN_DEBUG_SERVER;
-
 export const CleanAction = automodAction({
   configType: t.boolean,
   defaultConfig: false,
@@ -29,26 +27,13 @@ export const CleanAction = automodAction({
       }
     }
 
-    if (pluginData.guild.id === cleanDebugServer) {
-      const toDeleteFormatted = Array.from(messageIdsToDeleteByChannelId.entries())
-        .map(([channelId, messageIds]) => `- ${channelId}: ${messageIds.join(", ")}`)
-        .join("\n");
-      // tslint:disable-next-line:no-console
-      console.log(`[DEBUG] Cleaning messages (${ruleName}):\n${toDeleteFormatted}`);
-    }
-
     for (const [channelId, messageIds] of messageIdsToDeleteByChannelId.entries()) {
       for (const id of messageIds) {
         pluginData.state.logs.ignoreLog(LogType.MESSAGE_DELETE, id);
       }
 
       const channel = pluginData.guild.channels.cache.get(channelId as Snowflake) as TextChannel;
-      await channel.bulkDelete(messageIds as Snowflake[]).catch((err) => {
-        if (pluginData.guild.id === cleanDebugServer) {
-          // tslint:disable-next-line:no-console
-          console.error(`[DEBUG] Failed to bulk delete messages (${ruleName}): ${err}`);
-        }
-      });
+      await channel.bulkDelete(messageIds as Snowflake[]).catch(noop);
     }
   },
 });

backend/src/plugins/BotControl/commands/AddDashboardUserCmd.ts

@@ -1,14 +1,14 @@
 import { ApiPermissions } from "@shared/apiPermissions";
 import { TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { botControlCmd } from "../types";
 
 export const AddDashboardUserCmd = botControlCmd({
   trigger: ["add_dashboard_user"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/BotControl/commands/AddServerFromInviteCmd.ts

@@ -1,7 +1,7 @@
 import { ApiPermissions } from "@shared/apiPermissions";
 import { TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { DBDateFormat, isGuildInvite, isSnowflake, resolveInvite } from "../../../utils";
 import { botControlCmd } from "../types";
 import moment from "moment-timezone";

backend/src/plugins/BotControl/commands/AllowServerCmd.ts

@@ -1,7 +1,7 @@
 import { ApiPermissions } from "@shared/apiPermissions";
 import { TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { DBDateFormat, isSnowflake } from "../../../utils";
 import { botControlCmd } from "../types";
 import moment from "moment-timezone";
@@ -10,7 +10,7 @@ export const AllowServerCmd = botControlCmd({
   trigger: ["allow_server", "allowserver", "add_server", "addserver"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/BotControl/commands/ChannelToServerCmd.ts

@@ -1,6 +1,6 @@
 import { Guild, GuildChannel, TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { GuildInvite, isGuildInvite, resolveInvite, verboseUserMention } from "../../../utils";
 import { botControlCmd } from "../types";
 import { isEligible } from "../functions/isEligible";
@@ -9,7 +9,7 @@ export const ChannelToServerCmd = botControlCmd({
   trigger: ["channel_to_server", "channel2server"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/BotControl/commands/DisallowServerCmd.ts

@@ -1,6 +1,6 @@
 import { Snowflake, TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { noop } from "../../../utils";
 import { botControlCmd } from "../types";
 
@@ -8,7 +8,7 @@ export const DisallowServerCmd = botControlCmd({
   trigger: ["disallow_server", "disallowserver", "remove_server", "removeserver"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/BotControl/commands/LeaveServerCmd.ts

@@ -1,13 +1,13 @@
 import { Snowflake, TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { botControlCmd } from "../types";
 
 export const LeaveServerCmd = botControlCmd({
   trigger: ["leave_server", "leave_guild"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/BotControl/commands/ListDashboardPermsCmd.ts

@@ -2,7 +2,7 @@ import { TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
 import { AllowedGuild } from "../../../data/entities/AllowedGuild";
 import { ApiPermissionAssignment } from "../../../data/entities/ApiPermissionAssignment";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { resolveUser } from "../../../utils";
 import { botControlCmd } from "../types";
 

backend/src/plugins/BotControl/commands/ListDashboardUsersCmd.ts

@@ -1,6 +1,6 @@
 import { TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { resolveUser } from "../../../utils";
 import { botControlCmd } from "../types";
 

backend/src/plugins/BotControl/commands/ReloadGlobalPluginsCmd.ts

@@ -1,5 +1,5 @@
 import { TextChannel } from "discord.js";
-import { isOwnerPreFilter } from "../../../pluginUtils";
+import { isStaffPreFilter } from "../../../pluginUtils";
 import { getActiveReload, setActiveReload } from "../activeReload";
 import { botControlCmd } from "../types";
 
@@ -7,7 +7,7 @@ export const ReloadGlobalPluginsCmd = botControlCmd({
   trigger: "bot_reload_global_plugins",
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   async run({ pluginData, message }) {

backend/src/plugins/BotControl/commands/ReloadServerCmd.ts

@@ -1,13 +1,13 @@
 import { Snowflake, TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { botControlCmd } from "../types";
 
 export const ReloadServerCmd = botControlCmd({
   trigger: ["reload_server", "reload_guild"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/BotControl/commands/RemoveDashboardUserCmd.ts

@@ -1,13 +1,13 @@
 import { TextChannel } from "discord.js";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
+import { isStaffPreFilter, sendErrorMessage, sendSuccessMessage } from "../../../pluginUtils";
 import { botControlCmd } from "../types";
 
 export const RemoveDashboardUserCmd = botControlCmd({
   trigger: ["remove_dashboard_user"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/BotControl/commands/ServersCmd.ts

@@ -1,7 +1,7 @@
 import { TextChannel } from "discord.js";
 import escapeStringRegexp from "escape-string-regexp";
 import { commandTypeHelpers as ct } from "../../../commandTypes";
-import { isOwnerPreFilter } from "../../../pluginUtils";
+import { isStaffPreFilter } from "../../../pluginUtils";
 import { createChunkedMessage, getUser, sorter } from "../../../utils";
 import { botControlCmd } from "../types";
 
@@ -9,7 +9,7 @@ export const ServersCmd = botControlCmd({
   trigger: ["servers", "guilds"],
   permission: null,
   config: {
-    preFilters: [isOwnerPreFilter],
+    preFilters: [isStaffPreFilter],
   },
 
   signature: {

backend/src/plugins/GuildAccessMonitor/GuildAccessMonitorPlugin.ts

@@ -3,6 +3,8 @@ import * as t from "io-ts";
 import { BasePluginType, GlobalPluginData, typedGlobalEventListener } from "knub";
 import { AllowedGuilds } from "../../data/AllowedGuilds";
 import { zeppelinGlobalPlugin } from "../ZeppelinPluginBlueprint";
+import { env } from "../../env";
+import { Configs } from "../../data/Configs";
 
 interface GuildAccessMonitorPluginType extends BasePluginType {
   config: {};
@@ -15,7 +17,7 @@ async function checkGuild(pluginData: GlobalPluginData<GuildAccessMonitorPluginT
   if (!(await pluginData.state.allowedGuilds.isAllowed(guild.id))) {
     // tslint:disable-next-line:no-console
     console.log(`Non-allowed server ${guild.name} (${guild.id}), leaving`);
-    guild.leave();
+    // guild.leave();
   }
 }
 
@@ -35,8 +37,19 @@ export const GuildAccessMonitorPlugin = zeppelinGlobalPlugin<GuildAccessMonitorP
     }),
   ],
 
-  beforeLoad(pluginData) {
+  async beforeLoad(pluginData) {
     pluginData.state.allowedGuilds = new AllowedGuilds();
+
+    const defaultAllowedServers = env.DEFAULT_ALLOWED_SERVERS || [];
+    const configs = new Configs();
+    for (const serverId of defaultAllowedServers) {
+      if (!(await pluginData.state.allowedGuilds.isAllowed(serverId))) {
+        // tslint:disable-next-line:no-console
+        console.log(`Adding allowed-by-default server ${serverId} to the allowed servers`);
+        await pluginData.state.allowedGuilds.add(serverId);
+        await configs.saveNewRevision(`guild-${serverId}`, "plugins: {}", 0);
+      }
+    }
   },
 
   afterLoad(pluginData) {

backend/src/staff.ts

@@ -1,6 +1,8 @@
+import { env } from "./env";
+
 /**
  * Zeppelin staff have full access to the dashboard
  */
 export function isStaff(userId: string) {
-  return (process.env.STAFF ?? "").split(",").includes(userId);
+  return (env.STAFF ?? []).includes(userId);
 }

backend/src/utils/crypt.ts

@@ -1,21 +1,14 @@
 import { spawn, Worker, Pool } from "threads";
-import "../loadEnv";
 import type { CryptFns } from "./cryptWorker";
 import { MINUTES } from "../utils";
+import { env } from "../env";
 
-if (!process.env.KEY) {
-  // tslint:disable-next-line:no-console
-  console.error("Environment value KEY required for encryption");
-  process.exit(1);
-}
-
-const KEY = process.env.KEY;
 const pool = Pool(() => spawn(new Worker("./cryptWorker"), { timeout: 10 * MINUTES }), 8);
 
 export async function encrypt(data: string) {
-  return pool.queue((w) => w.encrypt(data, KEY));
+  return pool.queue((w) => w.encrypt(data, env.KEY));
 }
 
 export async function decrypt(data: string) {
-  return pool.queue((w) => w.decrypt(data, KEY));
+  return pool.queue((w) => w.decrypt(data, env.KEY));
 }

backend/tsconfig.json

@@ -24,5 +24,5 @@
     "useUnknownInCatchVariables": false,
     "allowJs": true
   },
-  "include": ["src/**/*.ts", "ormconfig.js"]
+  "include": ["src/**/*.ts"]
 }

dashboard/.env.example

@@ -1 +0,0 @@
-API_URL=

dashboard/webpack.config.js

@@ -1,5 +1,3 @@
-require("dotenv").config();
-
 const path = require("path");
 const VueLoaderPlugin = require("vue-loader/lib/plugin");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
@@ -154,7 +152,9 @@ let config = {
         js: ["./src/main.ts"],
       },
     }),
-    new DotenvPlugin(),
+    new DotenvPlugin({
+      path: path.resolve(process.cwd(), "../.env"),
+    }),
   ],
   resolve: {
     extensions: [".ts", ".tsx", ".js", ".mjs", ".vue"],

docker-compose.development.yml

@@ -0,0 +1,45 @@
+version: '3'
+name: zeppelin-dev
+volumes:
+  vscode-remote: {}
+  vscode-server: {}
+  jetbrains-data: {}
+services:
+  nginx:
+    build:
+      context: ./docker/development/nginx
+      args:
+        DOCKER_DEV_WEB_PORT: ${DOCKER_DEV_WEB_PORT:?Missing DOCKER_DEV_WEB_PORT}
+        API_PORT: ${API_PORT:?Missing API_PORT}
+    ports:
+      - "${DOCKER_DEV_WEB_PORT:?Missing DOCKER_DEV_WEB_PORT}:443"
+    volumes:
+      - ./:/zeppelin
+
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: ${DOCKER_DEV_MYSQL_ROOT_PASSWORD?:Missing DOCKER_DEV_MYSQL_ROOT_PASSWORD}
+      MYSQL_DATABASE: zeppelin
+      MYSQL_USER: zeppelin
+      MYSQL_PASSWORD: ${DOCKER_DEV_MYSQL_PASSWORD?:Missing DOCKER_DEV_MYSQL_PASSWORD}
+    ports:
+      - ${DOCKER_DEV_MYSQL_PORT:?Missing DOCKER_DEV_MYSQL_PORT}:3306
+    volumes:
+      - ./docker/development/data/mysql:/var/lib/mysql
+    command: --authentication-policy=mysql_native_password
+
+  devenv:
+    build:
+      context: ./docker/development/devenv
+      args:
+        DOCKER_DEV_SSH_PASSWORD: ${DOCKER_DEV_SSH_PASSWORD:?Missing DOCKER_DEV_SSH_PASSWORD}
+        DOCKER_DEV_UID: ${DOCKER_DEV_UID:-1000}
+    ports:
+      - "${DOCKER_DEV_SSH_PORT:?Missing DOCKER_DEV_SSH_PORT}:22"
+    volumes:
+      - ./:/home/ubuntu/zeppelin
+      - ~/.ssh:/home/ubuntu/.ssh
+      - vscode-remote:/home/ubuntu/.vscode-remote
+      - vscode-server:/home/ubuntu/.vscode-server
+      - jetbrains-data:/home/ubuntu/.cache/JetBrains

docker-compose.production.yml

@@ -0,0 +1,76 @@
+version: '3'
+name: zeppelin-prod
+services:
+  nginx:
+    build:
+      context: ./docker/production/nginx
+      args:
+        API_PORT: ${API_PORT:?Missing API_PORT}
+        DOCKER_PROD_DOMAIN: ${DOCKER_PROD_DOMAIN:?Missing DOCKER_PROD_DOMAIN}
+    ports:
+      - "${DOCKER_PROD_WEB_PORT:?Missing DOCKER_PROD_WEB_PORT}:443"
+    volumes:
+      - ./:/zeppelin
+
+  mysql:
+    image: mysql:8.0
+    environment:
+      MYSQL_ROOT_PASSWORD: ${DOCKER_PROD_MYSQL_ROOT_PASSWORD?:Missing DOCKER_PROD_MYSQL_ROOT_PASSWORD}
+      MYSQL_DATABASE: zeppelin
+      MYSQL_USER: zeppelin
+      MYSQL_PASSWORD: ${DOCKER_PROD_MYSQL_PASSWORD?:Missing DOCKER_PROD_MYSQL_PASSWORD}
+    ports:
+      - ${DOCKER_PROD_MYSQL_PORT:?Missing DOCKER_PROD_MYSQL_PORT}:3306
+    volumes:
+      - ./docker/production/data/mysql:/var/lib/mysql
+    command: --authentication-policy=mysql_native_password
+
+  prepare_backend:
+    build:
+      context: ./docker/production/node
+      args:
+        DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID}
+        DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID}
+    depends_on:
+      - mysql
+    volumes:
+      - ./:/zeppelin
+    command: |-
+      bash -c "cd /zeppelin/backend && npm ci && npm run build && npm run migrate-prod"
+
+  api:
+    build:
+      context: ./docker/production/node
+      args:
+        DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID}
+        DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID}
+    restart: on-failure
+    depends_on:
+      - prepare_backend
+    volumes:
+      - ./:/zeppelin
+    command: ["/bin/bash", "/zeppelin/docker/production/start-api.sh"]
+
+  bot:
+    build:
+      context: ./docker/production/node
+      args:
+        DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID}
+        DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID}
+    restart: on-failure
+    depends_on:
+      - prepare_backend
+    volumes:
+      - ./:/zeppelin
+    command: ["/bin/bash", "/zeppelin/docker/production/start-bot.sh"]
+
+  build_dashboard:
+    build:
+      context: ./docker/production/node
+      args:
+        DOCKER_USER_UID: ${DOCKER_USER_UID:?Missing DOCKER_USER_UID}
+        DOCKER_USER_GID: ${DOCKER_USER_GID:?Missing DOCKER_USER_GID}
+    volumes:
+      - ./:/zeppelin
+    command: |-
+      bash -c "cd /zeppelin/dashboard && npm ci && npm run build"

docker/development/data/mysql/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

docker/development/devenv/Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:20.04
+
+ARG DOCKER_DEV_UID
+ARG DOCKER_DEV_SSH_PASSWORD
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=UTC
+
+# Set up some core packages
+RUN apt-get update
+RUN apt-get install -y sudo git curl
+
+# Set up SSH access
+RUN apt-get install -y openssh-server iptables
+RUN mkdir /var/run/sshd
+RUN useradd -rm -d /home/ubuntu -s /bin/bash -g root -G sudo -u $DOCKER_DEV_UID ubuntu
+RUN echo "ubuntu:${DOCKER_DEV_SSH_PASSWORD}" | chpasswd
+
+# Set up proper permissions for volumes
+RUN mkdir -p /home/ubuntu/zeppelin /home/ubuntu/.vscode-remote /home/ubuntu/.vscode-server /home/ubuntu/.cache/JetBrains
+RUN chown -R ubuntu /home/ubuntu
+
+# Install Node.js 16 and packages needed to build native packages
+RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -
+RUN apt-get install -y nodejs gcc g++ make python3
+
+CMD /usr/sbin/sshd -D -e

docker/development/nginx/Dockerfile

@@ -0,0 +1,11 @@
+FROM nginx
+
+ARG API_PORT
+ARG DOCKER_DEV_API_PORT
+ARG DOCKER_DEV_DASHBOARD_PORT
+
+RUN apt-get update && apt-get install -y openssl
+RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/localhost-cert.key -out /etc/ssl/certs/localhost-cert.pem -days 3650 -subj '/CN=localhost' -nodes
+
+COPY ./default.conf /etc/nginx/conf.d/default.conf
+RUN sed -ir "s/_API_PORT_/${API_PORT}/g" /etc/nginx/conf.d/default.conf

docker/development/nginx/default.conf

@@ -0,0 +1,39 @@
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name localhost;
+
+  root /zeppelin/dashboard/dist;
+
+  location / {
+    index index.html;
+    try_files $uri $uri/ /index.html;
+  }
+
+  # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable
+  set $backend_upstream "http://devenv:_API_PORT_";
+
+  location /api {
+    # Remove /api/ from the beginning when passing the path to the API process
+    rewrite /api(/.*)$ $1 break;
+
+    # Using a variable in proxy_pass also requires resolver to be set.
+    # This is the address of the internal docker compose DNS server.
+    resolver 127.0.0.11;
+
+    proxy_pass $backend_upstream$uri$is_args$args;
+    proxy_redirect off;
+
+    client_max_body_size 200M;
+  }
+
+  ssl_certificate /etc/ssl/certs/localhost-cert.pem;
+  ssl_certificate_key /etc/ssl/private/localhost-cert.key;
+
+  ssl_session_timeout 1d;
+  ssl_session_cache shared:MozSSL:10m;
+  ssl_session_tickets off;
+
+  ssl_protocols TLSv1.3;
+  ssl_prefer_server_ciphers off;
+}

docker/production/config/mysql.conf.d/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

docker/production/data/mysql/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

docker/production/nginx/Dockerfile

@@ -0,0 +1,11 @@
+FROM nginx
+
+ARG API_PORT
+ARG DOCKER_PROD_DOMAIN
+
+RUN apt-get update && apt-get install -y openssl
+RUN openssl req -x509 -newkey rsa:4096 -keyout /etc/ssl/private/zeppelin-self-signed-cert.key -out /etc/ssl/certs/zeppelin-self-signed-cert.pem -days 3650 -subj "/CN=${DOCKER_PROD_DOMAIN}" -nodes
+
+COPY ./default.conf /etc/nginx/conf.d/default.conf
+RUN sed -ir "s/_API_PORT_/${API_PORT}/g" /etc/nginx/conf.d/default.conf
+RUN sed -ir "s/_DOCKER_PROD_DOMAIN_/${DOCKER_PROD_DOMAIN}/g" /etc/nginx/conf.d/default.conf

docker/production/nginx/default.conf

@@ -0,0 +1,39 @@
+server {
+  listen 443 ssl http2;
+  listen [::]:443 ssl http2;
+  server_name _DOCKER_PROD_DOMAIN_;
+
+  root /zeppelin/dashboard/dist;
+
+  location / {
+    index index.html;
+    try_files $uri $uri/ /index.html;
+  }
+
+  # Using a variable here stops nginx from crashing if the dev container is restarted or becomes otherwise unavailable
+  set $backend_upstream "http://api:_API_PORT_";
+
+  location /api {
+    # Remove /api/ from the beginning when passing the path to the API process
+    rewrite /api(/.*)$ $1 break;
+
+    # Using a variable in proxy_pass also requires resolver to be set.
+    # This is the address of the internal docker compose DNS server.
+    resolver 127.0.0.11;
+
+    proxy_pass $backend_upstream$uri$is_args$args;
+    proxy_redirect off;
+
+    client_max_body_size 200M;
+  }
+
+  ssl_certificate /etc/ssl/certs/zeppelin-self-signed-cert.pem;
+  ssl_certificate_key /etc/ssl/private/zeppelin-self-signed-cert.key;
+
+  ssl_session_timeout 1d;
+  ssl_session_cache shared:MozSSL:10m;
+  ssl_session_tickets off;
+
+  ssl_protocols TLSv1.3;
+  ssl_prefer_server_ciphers off;
+}

docker/production/node/Dockerfile

@@ -0,0 +1,10 @@
+FROM node:16.16
+
+ARG DOCKER_USER_UID
+ARG DOCKER_USER_GID
+
+# This custom Dockerfile is needed for the Node image so we can change the uid/gid used for the node user
+# See https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md#non-root-user
+RUN groupmod -g "${DOCKER_USER_GID}" node && usermod -u "${DOCKER_USER_UID}" -g "${DOCKER_USER_GID}" node
+
+USER node

docker/production/start-api.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This wrapper script is used for two purposes:
+# 1. Waiting for the prepare_backend container to finish before starting (see https://github.com/docker/compose/issues/5007#issuecomment-335815508)
+# 2. Forwarding signals to the app (see https://unix.stackexchange.com/a/196053)
+
+# Wait for the backend preparations to finish before continuing
+echo "Waiting for prepare_backend to finish before starting the API..."
+while ping -c1 prepare_backend &>/dev/null; do sleep 1; done;
+
+echo "Starting the API"
+cd /zeppelin/backend
+exec npm run start-api-prod

docker/production/start-bot.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This wrapper script is used for two purposes:
+# 1. Waiting for the prepare_backend container to finish before starting (see https://github.com/docker/compose/issues/5007#issuecomment-335815508)
+# 2. Forwarding signals to the app (see https://unix.stackexchange.com/a/196053)
+
+# Wait for the backend preparations to finish before continuing
+echo "Waiting for prepare_backend to finish before starting the bot..."
+while ping -c1 prepare_backend &>/dev/null; do sleep 1; done;
+
+echo "Starting the bot"
+cd /zeppelin/backend
+exec npm run start-bot-prod

process-api.json

@@ -1,12 +0,0 @@
-{
-  "apps": [
-    {
-      "name": "zeppelin-api",
-      "cwd": "./backend",
-      "script": "npm",
-      "args": "run start-api-prod",
-      "log_date_format": "YYYY-MM-DD HH:mm:ss.SSS",
-      "exp_backoff_restart_delay": 2500
-    }
-  ]
-}

process-bot.json

@@ -1,12 +0,0 @@
-{
-  "apps": [
-    {
-      "name": "zeppelin",
-      "cwd": "./backend",
-      "script": "npm",
-      "args": "run start-bot-prod",
-      "log_date_format": "YYYY-MM-DD HH:mm:ss.SSS",
-      "exp_backoff_restart_delay": 2500
-    }
-  ]
-}

update-backend-hotfix.sh

@@ -1,16 +0,0 @@
-#!/bin/bash
-
-# Load nvm
-. ~/.nvm/nvm.sh
-
-# Run hotfix update
-cd backend
-nvm use
-git pull
-npm run build
-
-# Restart processes
-cd ..
-nvm use
-pm2 restart process-bot.json
-pm2 restart process-api.json

update-backend.sh

@@ -1,25 +0,0 @@
-#!/bin/bash
-
-# Load nvm
-. ~/.nvm/nvm.sh
-
-# Stop current processes
-nvm use
-pm2 delete process-bot.json
-pm2 delete process-api.json
-
-# Run update
-nvm use
-git pull
-npm ci
-
-cd backend
-npm ci
-npm run build
-npm run migrate-prod
-
-# Start processes again
-cd ..
-nvm use
-pm2 start process-bot.json
-pm2 start process-api.json

update-dashboard.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-TARGET_DIR=/var/www/zeppelin.gg
-
-# Load nvm
-. ~/.nvm/nvm.sh
-
-# Update dashboard
-cd dashboard
-git pull
-nvm use
-npm ci
-npm run build
-rm -r $TARGET_DIR/*
-cp -R dist/* $TARGET_DIR
-
-# Return
-cd ..

update.sh

@@ -1,4 +1,9 @@
 #!/bin/bash
 
-. ./update-backend.sh
-. ./update-dashboard.sh
+echo Updating Zeppelin...
+
+docker compose -f docker-compose.production.yml stop
+git pull
+docker compose -f docker-compose.production.yml start
+
+echo Update finished!