Server: Improved config and support for Docker

.env-sample

@@ -1,9 +1,26 @@
-# Example of local config, for development:
+# =============================================================================
+# PRODUCTION CONFIG EXAMPLE
+# -----------------------------------------------------------------------------
+# By default it will use SQLite, but that's mostly to test and evaluate the
+# server. So you'll want to specify db connection settings to use Postgres.
+# =============================================================================
 #
-# JOPLIN_BASE_URL=http://localhost:22300
-# JOPLIN_PORT=22300
+# APP_BASE_URL=https://example.com/joplin
+# APP_PORT=22300
+# 
+# DB_CLIENT=pg
+# POSTGRES_PASSWORD=joplin
+# POSTGRES_DATABASE=joplin
+# POSTGRES_USER=joplin
+# POSTGRES_PORT=5432
+# POSTGRES_HOST=localhost
 
-# Example of config for production:
+# =============================================================================
+# DEV CONFIG EXAMPLE
+# -----------------------------------------------------------------------------
+# Example of local config, for development. In dev mode, you would usually use
+# SQLite so database settings are not needed.
+# =============================================================================
 #
-# JOPLIN_BASE_URL=https://example.com/joplin
-# JOPLIN_PORT=22300
+# APP_BASE_URL=http://localhost:22300
+# APP_PORT=22300

Dockerfile.server

@@ -16,8 +16,15 @@ WORKDIR /home/$user
 
 RUN mkdir /home/$user/logs
 
+# Install the root scripts but don't run postinstall (which would bootstrap
+# and build TypeScript files, but we don't have the TypeScript files at
+# this point)
+
+COPY --chown=$user:$user package*.json ./
+RUN npm install --ignore-scripts
+
 # To take advantage of the Docker cache, we first copy all the package.json
-# and package-lock.json files, as they rarely change? and then bootstrap
+# and package-lock.json files, as they rarely change, and then bootstrap
 # all the packages.
 #
 # Note that bootstrapping the packages will run all the postinstall
@@ -27,19 +34,10 @@ RUN mkdir /home/$user/logs
 # We can't run boostrap with "--ignore-scripts" because that would
 # prevent certain sub-packages, such as sqlite3, from being built
 
-COPY --chown=$user:$user package*.json ./
-
-# Install the root scripts but don't run postinstall (which would bootstrap
-# and build TypeScript files, but we don't have the TypeScript files at
-# this point)
-
-RUN npm install --ignore-scripts
-
 COPY --chown=$user:$user packages/fork-sax/package*.json ./packages/fork-sax/
-COPY --chown=$user:$user packages/lib/package*.json ./packages/lib/
 COPY --chown=$user:$user packages/renderer/package*.json ./packages/renderer/
 COPY --chown=$user:$user packages/tools/package*.json ./packages/tools/
-COPY --chown=$user:$user packages/server/package*.json ./packages/server/
+COPY --chown=$user:$user packages/lib/package*.json ./packages/lib/
 COPY --chown=$user:$user lerna.json .
 COPY --chown=$user:$user tsconfig.json .
 
@@ -50,22 +48,29 @@ COPY --chown=$user:$user packages/turndown ./packages/turndown
 COPY --chown=$user:$user packages/turndown-plugin-gfm ./packages/turndown-plugin-gfm
 COPY --chown=$user:$user packages/fork-htmlparser2 ./packages/fork-htmlparser2
 
-RUN ls -la /home/$user
-
 # Then bootstrap only, without compiling the TypeScript files
 
 RUN npm run bootstrap
 
+# We have a separate step for the server files because they are more likely to
+# change.
+
+COPY --chown=$user:$user packages/server/package*.json ./packages/server/
+RUN npm run bootstrapServerOnly
+
+# Now copy the source files. Put lib and server last as they are more likely to change.
+
 COPY --chown=$user:$user packages/fork-sax ./packages/fork-sax
-COPY --chown=$user:$user packages/lib ./packages/lib
 COPY --chown=$user:$user packages/renderer ./packages/renderer
 COPY --chown=$user:$user packages/tools ./packages/tools
+COPY --chown=$user:$user packages/lib ./packages/lib
 COPY --chown=$user:$user packages/server ./packages/server
 
 # Finally build everything, in particular the TypeScript files.
 
 RUN npm run build
 
-EXPOSE ${JOPLIN_PORT}
+ENV RUNNING_IN_DOCKER=1
+EXPOSE ${APP_PORT}
 
 CMD [ "npm", "--prefix", "packages/server", "start" ]

docker-compose.db-dev.yml

@@ -0,0 +1,15 @@
+# For development this compose file starts the database only. The app can then
+# be started using `npm run start-dev`, which is useful for development, because
+# it means the app Docker file doesn't have to be rebuilt on each change.
+
+version: '3'
+
+services:
+    db:
+        image: postgres:13.1
+        ports:
+            - "5432:5432"
+        environment:
+            - POSTGRES_PASSWORD=joplin
+            - POSTGRES_USER=joplin
+            - POSTGRES_DB=joplin

docker-compose.server-dev.yml

@@ -1,28 +1,27 @@
-# For development, the easiest might be to only start the Postgres container and
-# run the app directly with `npm start`. Or use sqlite3.
+# This compose file can be used in development to run both the database and app
+# within Docker.
 
 version: '3'
 
 services:
-    # app:
-    #   build:
-    #     context: .
-    #     dockerfile: Dockerfile.server-dev
-    #   ports:
-    #     - "22300:22300"
-    #   # volumes:
-    #   #   - ./packages/server/:/var/www/joplin/packages/server/
-    #   #   - /var/www/joplin/packages/server/node_modules/
-    db:
+    app:
         build:
             context: .
-            dockerfile: Dockerfile.db
+            dockerfile: Dockerfile.server
+        ports:
+            - "22300:22300"
+        environment:
+            - DB_CLIENT=pg
+            - POSTGRES_PASSWORD=joplin
+            - POSTGRES_DATABASE=joplin
+            - POSTGRES_USER=joplin
+            - POSTGRES_PORT=5432
+            - POSTGRES_HOST=localhost
+    db:
+        image: postgres:13.1
         ports:
             - "5432:5432"
         environment:
-            # TODO: Considering the database is only exposed to the
-            # application, and not to the outside world, is there a need to
-            # pick a secure password?
             - POSTGRES_PASSWORD=joplin
             - POSTGRES_USER=joplin
-            - POSTGRES_DB=joplin
+            - POSTGRES_DB=joplin

docker-compose.server.yml

@@ -1,40 +1,34 @@
+# This is a sample docker-compose file that can be used to run Joplin Server
+# along with a PostgreSQL server.
+#
+# All environment variables are optional. If you don't set them, you will get a
+# warning from docker-compose, however the app should use working defaults.
+
 version: '3'
 
 services:
-    app:
-        environment:
-            - JOPLIN_BASE_URL=${JOPLIN_BASE_URL}
-            - JOPLIN_PORT=${JOPLIN_PORT}
-        restart: unless-stopped
-        build:
-            context: .
-            dockerfile: Dockerfile.server
-        ports:
-            - "${JOPLIN_PORT}:${JOPLIN_PORT}"
-        # volumes:
-        #     # Mount the server directory so that it's possible to edit file
-        #     # while the container is running. However don't mount the
-        #     # node_modules directory which will be specific to the Docker
-        #     # image (eg native modules will be built for Ubuntu, while the
-        #     # container might be running in Windows)
-        #     # https://stackoverflow.com/a/37898591/561309
-        #     - ./packages/server:/home/joplin/packages/server
-        #     - /home/joplin/packages/server/node_modules/
     db:
-        restart: unless-stopped
-        # By default, the Postgres image saves the data to a Docker volume,
-        # so it persists whenever the server is restarted using
-        # `docker-compose up`. Note that it would however be deleted when
-        # running `docker-compose down`.
-        build:
-            context: .
-            dockerfile: Dockerfile.db
+        image: postgres:13.1
         ports:
             - "5432:5432"
+        restart: unless-stopped
+        environment:
+            - APP_PORT=22300
+            - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+            - POSTGRES_USER=${POSTGRES_USER}
+            - POSTGRES_DB=${POSTGRES_DATABASE}
+    app:
+        image: joplin/server:latest
+        depends_on:
+            - db
+        ports:
+            - "22300:22300"
+        restart: unless-stopped
         environment:
-            # TODO: Considering the database is only exposed to the
-            # application, and not to the outside world, is there a need to
-            # pick a secure password?
-            - POSTGRES_PASSWORD=joplin
-            - POSTGRES_USER=joplin
-            - POSTGRES_DB=joplin
+            - APP_BASE_URL=${APP_BASE_URL}
+            - DB_CLIENT=pg
+            - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+            - POSTGRES_DATABASE=${POSTGRES_DATABASE}
+            - POSTGRES_USER=${POSTGRES_USER}
+            - POSTGRES_PORT=${POSTGRES_PORT}
+            - POSTGRES_HOST=db

package.json

@@ -8,6 +8,7 @@
   "license": "MIT",
   "scripts": {
     "bootstrap": "lerna bootstrap --no-ci",
+    "bootstrapServerOnly": "lerna bootstrap --no-ci --include-dependents --include-dependencies --scope @joplin/server",
     "bootstrapIgnoreScripts": "lerna bootstrap --ignore-scripts --no-ci",
     "build": "lerna run build && npm run tsc",
     "buildApiDoc": "npm start --prefix=packages/app-cli -- apidoc ../../readme/api/references/rest_api.md",

packages/server/README.md

@@ -4,64 +4,86 @@
 
 First copy `.env-sample` to `.env` and edit the values in there:
 
-- `JOPLIN_BASE_URL`: This is the base public URL where the service will be running. For example, if you want it to run from `https://example.com/joplin`, this is what you should set the URL to. The base URL can include the port.
-- `JOPLIN_PORT`: The local port on which the Docker container will listen. You would typically map this port to 443 (TLS) with a reverse proxy.
+- `APP_BASE_URL`: This is the base public URL where the service will be running. For example, if you want it to run from `https://example.com/joplin`, this is what you should set the URL to. The base URL can include the port.
+- `APP_PORT`: The local port on which the Docker container will listen. You would typically map this port to 443 (TLS) with a reverse proxy.
 
-## Install application
+## Running the server
+
+To start the server with default configuration, run:
 
 ```shell
-wget https://github.com/laurent22/joplin/archive/server-v1.6.4.tar.gz
-tar xzvf server-v1.6.4.tar.gz
-mv joplin-server-v1.6.4 joplin-server
-cd joplin-server
-docker-compose --file docker-compose.server.yml up --detach
+docker run --env-file .env -p 22300:22300 joplin/server:latest
+```
+
+This will start the server, which will listen on port **22300** on **localhost**. By default it will use SQLite, which allows you to test the app without setting up a database. To run it for production though, you'll want to connect the container to a database, as described below.
+
+## Setup the database
+
+You can setup the container to either use an existing PostgreSQL server, or connect it to a new one using docker-compose
+
+### Using an existing PostgreSQL server
+
+To use an existing PostgresSQL server, set the following environment variables in the .env file:
+
+```conf
+DB_CLIENT=pg
+POSTGRES_PASSWORD=joplin
+POSTGRES_DATABASE=joplin
+POSTGRES_USER=joplin
+POSTGRES_PORT=5432
+POSTGRES_HOST=localhost
 ```
 
-This will start the server, which will listen on port **22300** on **localhost**.
+Make sure that the provided database and user exist as the server will not create them.
+
+### Using docker-compose
 
-Due to the restart policy defined in the docker-compose file, the server will be restarted automatically whenever the host reboots.
+A [sample docker-compose file](https://github.com/laurent22/joplin/blob/dev/docker-compose.server.yml
+ ) is available to show how to use Docker to install both the database and server and connect them:
 
 ## Setup reverse proxy
 
-You will then need to expose this server to the internet by setting up a reverse proxy, and that will depend on how your server is currently configured, and whether you already have Nginx or Apache running:
+Once Joplin Server is running, you will then need to expose it to the internet by setting up a reverse proxy, and that will depend on how your server is currently configured, and whether you already have Nginx or Apache running:
 
 - [Apache Reverse Proxy](https://httpd.apache.org/docs/current/mod/mod_proxy.html)
 - [Nginx Reverse Proxy](https://docs.nginx.com/nginx/admin-guide/web-server/reverse-proxy/)
 
-## Setup admin user
+## Setup the website
 
-For the following instructions, we'll assume that the Joplin server is running on `https://example.com/joplin`.
+Once the server is exposed to the internet, you can open the admin UI and get it ready for synchronisation. For the following instructions, we'll assume that the Joplin server is running on `https://example.com/joplin`.
 
-By default, the instance will be setup with an admin user with email **admin@localhost** and password **admin** and you should change this by opening the admin UI. To do so, open `https://example.com/joplin/login`. From there, go to Profile and change the admin password.
+### Secure the admin user
 
-## Setup a user for sync
+By default, the instance will be setup with an admin user with email **admin@localhost** and password **admin** and you should change this. To do so, open `https://example.com/joplin/login` and login as admin. Then go to the Profile section and change the admin password.
 
-While the admin user can be used for synchronisation, it is recommended to create a separate non-admin user for it. To do, open the admin UI and navigate to the Users page - from there you can create a new user.
+### Create a user for sync
 
-Once this is done, you can use the email and password you specified to sync this user account with your Joplin clients.
+While the admin user can be used for synchronisation, it is recommended to create a separate non-admin user for it. To do so, navigate to the Users page - from there you can create a new user. Once this is done, you can use the email and password you specified to sync this user account with your Joplin clients.
 
 ## Checking the logs
 
 Checking the log can be done the standard Docker way:
 
-```shell
+```bash
+# With Docker:
+docker logs --follow CONTAINER
+
+# With docker-compose:
 docker-compose --file docker-compose.server.yml logs
 ```
 
-# Set up for development
+# Setup for development
 
-## Setting up the database
+## Setup up the database
 
 ### SQLite
 
-The server supports SQLite for development and test units. To use it, open `src/config-dev.ts` and uncomment the sqlite3 config.
+By default the server supports SQLite for development, so nothing needs to be setup.
 
 ### PostgreSQL
 
-It's best to use PostgreSQL as this is what is used in production, however it requires Docker.
-
-To use it, from the monorepo root, run `docker-compose --file docker-compose.server-dev.yml up`, which will start the PostgreSQL database.
+To use Postgres, from the monorepo root, run `docker-compose --file docker-compose.server-dev.yml up`, which will start the PostgreSQL database.
 
 ## Starting the server
 
-From `packages/server`, run `npm run start-dev`
+From `packages/server`, run `npm run start-dev`

packages/server/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@joplin/server",
-  "version": "1.7.0",
+  "version": "1.7.1",
   "private": true,
   "scripts": {
     "start-dev": "nodemon --config nodemon.json dist/app.js --env dev",
@@ -26,6 +26,7 @@
     "markdown-it": "^12.0.4",
     "mustache": "^3.1.0",
     "nanoid": "^2.1.1",
+    "node-env-file": "^0.1.8",
     "nodemon": "^2.0.6",
     "pg": "^8.5.1",
     "query-string": "^6.8.3",