diff --git a/Justfile b/Justfile
index 6e287a87a2..f38977bbd6 100644
--- a/Justfile
+++ b/Justfile
@@ -1,4 +1,4 @@
-# Copyright (c) 2020, 2021 Humanitarian OpenStreetMap Team
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
#
# This file is part of FMTM.
#
@@ -16,55 +16,29 @@
# along with FMTM. If not, see .
#
-# Builds
+mod start 'contrib/just/start/Justfile'
+mod stop 'contrib/just/stop/Justfile'
+mod build 'contrib/just/build/Justfile'
+mod test 'contrib/just/test/Justfile'
+mod dotenv 'contrib/just/dotenv/Justfile'
-build-backend:
- docker compose build api
+# Run the help script
+default:
+ @just --unstable help
-build-frontend:
- docker compose build ui
+# View available commands
+help:
+ @just --unstable --list --justfile {{justfile()}}
-build: build-backend build-frontend
+# Run database migrations for backend
+migrate:
+ docker compose up -d migrations
-# Run
-
-run:
- docker compose up -d
-
-run-without-central:
- docker compose --profile no-odk up -d
-
-run-with-josm:
- docker compose \
- -f docker-compose.yml \
- -f contrib/josm/docker-compose.yml \
- up -d
-
-run-with-tunnels:
- docker compose \
- -f docker-compose.yml \
- -f contrib/tunnel/fmtm/docker-compose.yml \
- -f contrib/tunnel/odk/docker-compose.yml \
- up -d
-
-stop:
- docker compose down
-
-clean-db:
+# Delete local database, S3, and ODK Central data
+clean:
docker compose down -v
-# Tests
-
-test-backend:
- docker compose run --rm api pytest
-
-test-frontend:
- docker compose run --rm ui-test
-
-test: test-backend test-frontend
-
-# Maintenance
-
+# Run pre-commit hooks
lint:
TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
--volume $PWD:$PWD --workdir $PWD \
@@ -72,6 +46,7 @@ lint:
'git config --global --add safe.directory $PWD \
&& pre-commit run --all-files'
+# Increment version
bump:
TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
--volume $PWD:$PWD --workdir $PWD \
@@ -82,33 +57,35 @@ bump:
&& cd src/backend \
&& cz bump --check-consistency'
-# Docs
-
-docs-rebuild: docs-clean docs-doxygen docs-uml
-
-docs-clean:
- @rm -rf docs/{apidocs,html,docbook,man} docs/packages.png docs/classes.png
-
-docs-doxygen:
- cd docs && doxygen
+# Run docs website locally
+docs:
+ @echo
+ @echo "\033[0;33m ############################################### \033[0m"
+ @echo
+ @echo
+ @echo "\033[0;34m Access the docs site on: http://localhost:55425 \033[0m"
+ @echo
+ @echo
+ @echo "\033[0;33m ############################################### \033[0m"
+ @echo
-docs-uml:
- cd docs && pyreverse -o png ../src/backend/app
-
-docs-pdf:
- # Strip any unicode out of the markdown file before converting to PDF
- # FIXME
- MDS := \
- docs/dev/Backend.md \
- docs/dev/Database-Tips.md \
- docs/dev/Release-Cycle.md \
- docs/dev/Frontend.md \
- docs/dev/Production.md \
- docs/dev/Version-Control.md \
- docs/dev/Setup.md \
- docs/dev/Troubleshooting.md \
- PDFS := $(MDS:.md=.pdf)
- @echo "Converting $PDFS to a PDF"
- @new=$(notdir $(basename $PDFS)); \
- iconv -f utf-8 -t US $PDFS -c | \
- pandoc $PDFS -f markdown -t pdf -s -o /tmp/$$new.pdf
\ No newline at end of file
+ TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
+ --volume $PWD:$PWD --workdir $PWD --publish 55425:3000 \
+ --entrypoint='sh -c' api \
+ 'git config --global --add safe.directory $PWD \
+ && mkdocs serve --dev-addr 0.0.0.0:3000'
+
+# Mount an S3 bucket on your filesystem
+mount-s3:
+ #!/usr/bin/env sh
+ fstab_entry="fmtm-data /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\
+ use_path_request_style,passwd_file=/home/$(whoami)/s3-creds/fmtm-local,\
+ url=http://s3.fmtm.localhost:7050 0 0"
+
+ if ! grep -q "$fstab_entry" /etc/fstab; then
+ echo "Mounting local FMTM S3 permanently in /etc/fstab"
+ echo "$fstab_entry" | sudo tee -a /etc/fstab > /dev/null
+ echo
+ else
+ echo "Local FMTM S3 is already mounted"
+ fi
diff --git a/contrib/josm/docker-compose.yml b/contrib/josm/docker-compose.yml
index a105874522..9fbe243caf 100644
--- a/contrib/josm/docker-compose.yml
+++ b/contrib/josm/docker-compose.yml
@@ -22,7 +22,7 @@ services:
josm:
image: "ghcr.io/hotosm/fmtm/josm:latest"
build:
- context: .
+ context: contrib/josm
container_name: josm
environment:
- DISPLAY=josm-novnc:0.0
@@ -32,13 +32,13 @@ services:
- fmtm-net
- x11
ports:
- - 8111:80
+ - "8111:80"
restart: "unless-stopped"
josm-novnc:
image: "ghcr.io/hotosm/fmtm/josm-novnc:latest"
build:
- context: novnc
+ context: contrib/josm/novnc
container_name: josm_novnc
environment:
- DISPLAY_WIDTH=1280
diff --git a/contrib/just/README.md b/contrib/just/README.md
new file mode 100644
index 0000000000..9045bbe80c
--- /dev/null
+++ b/contrib/just/README.md
@@ -0,0 +1,13 @@
+# Just Submodules
+
+- This directory contains submodules for the parent Justfile.
+- It allows for submodules to be used like:
+
+```bash
+just build frontend
+just build backend
+
+# Instead of (only top level)
+just build-backend
+just build-frontend
+```
diff --git a/contrib/tunnel/odk/docker-compose.yml b/contrib/just/dotenv/Justfile
similarity index 64%
rename from contrib/tunnel/odk/docker-compose.yml
rename to contrib/just/dotenv/Justfile
index 77b7734197..5d55a4f94e 100644
--- a/contrib/tunnel/odk/docker-compose.yml
+++ b/contrib/just/dotenv/Justfile
@@ -1,4 +1,5 @@
-# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
+#
# This file is part of FMTM.
#
# FMTM is free software: you can redistribute it and/or modify
@@ -15,17 +16,19 @@
# along with FMTM. If not, see .
#
-networks:
- fmtm-net:
- name: fmtm-${GIT_BRANCH:-local}
+# Update a variable in the .env file
+[no-cd]
+update key value:
+ #!/usr/bin/env sh
+
+ var_name={{key}}
+ var_value={{value}}
+ var_pattern="^${var_name}="
+ new_var="${var_name}=${var_value}"
+
+ if grep -Eq "${var_pattern}" .env; then
+ sed -i "s|${var_pattern}.*|${new_var}|" .env
+ else
+ echo "${new_var}" >> .env
+ fi
-services:
- central-tunnel:
- image: "docker.io/cloudflare/cloudflared:latest"
- depends_on:
- central:
- condition: service_healthy
- networks:
- - fmtm-net
- restart: "unless-stopped"
- command: tunnel --url http://central:8383
diff --git a/contrib/just/start/Justfile b/contrib/just/start/Justfile
new file mode 100644
index 0000000000..262e7aeb2c
--- /dev/null
+++ b/contrib/just/start/Justfile
@@ -0,0 +1,140 @@
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of FMTM.
+#
+# FMTM is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# FMTM is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with FMTM. If not, see .
+#
+
+# Start FMTM
+[no-cd]
+default:
+ docker compose up -d
+
+# Start backend API only
+[no-cd]
+backend:
+ docker compose up -d api
+
+# Start frontend UI only
+[no-cd]
+frontend:
+ docker compose up -d ui
+
+# Start FMTM without ODK Central
+[no-cd]
+without-central:
+ docker compose --profile no-odk up -d
+
+# Start FMTM with JOSM
+[no-cd]
+josm:
+ docker compose \
+ -f docker-compose.yml \
+ -f contrib/josm/docker-compose.yml \
+ up -d
+
+ @echo
+ @echo "\033[0;33m ############################################### \033[0m"
+ @echo
+ @echo " Access the JOSM Remote API: http://localhost:8111"
+ @echo " Access the JOSM GUI in browser: http://localhost:8112"
+ @echo
+ @echo "\033[0;33m ############################################### \033[0m"
+ @echo
+
+# Start FMTM with tunnels
+[no-cd]
+tunnel:
+ #!/usr/bin/env sh
+
+ docker compose \
+ -f docker-compose.yml \
+ -f contrib/tunnel/docker-compose.yml \
+ up -d --wait
+
+ # Workaround to until PR merged:
+ # https://github.com/cloudflare/cloudflared/pull/1135
+ # Wait until services ready without HEALTHCHECK
+ sleep 5
+
+ fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
+ api_url=$(just --unstable start _get-tunnel-url 'api')
+ odk_url=$(just --unstable start _get-tunnel-url 'central')
+ s3_url=$(just --unstable start _get-tunnel-url 's3')
+
+ just --unstable dotenv update "EXTRA_CORS_ORIGINS" "${fmtm_url}"
+ just --unstable dotenv update "S3_ENDPOINT" "${s3_url}"
+
+ # Restart the API and UI with environment variables set
+ API_URL_OVERRIDE="${api_url}" docker compose \
+ -f docker-compose.yml \
+ -f contrib/tunnel/docker-compose.yml \
+ up -d api ui
+
+ # Restart ODK Central with domain override (for form download urls)
+ CENTRAL_DOMAIN_OVERRIDE="$(echo "${odk_url}" | sed 's|^https://||')" \
+ docker compose \
+ -f docker-compose.yml \
+ -f contrib/tunnel/docker-compose.yml \
+ up -d central
+
+ just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"
+
+# View the URLs for created tunnels
+[no-cd]
+view-tunnel-urls:
+ #!/usr/bin/env sh
+
+ fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
+ api_url=$(just --unstable start _get-tunnel-url 'api')
+ odk_url=$(just --unstable start _get-tunnel-url 'central')
+ s3_url=$(just --unstable start _get-tunnel-url 's3')
+
+ just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"
+
+[no-cd]
+_get-tunnel-url service_name:
+ #!/usr/bin/env sh
+
+ service_url=$(docker compose \
+ -f docker-compose.yml \
+ -f contrib/tunnel/docker-compose.yml \
+ logs {{service_name}}-tunnel | \
+ grep 'Your quick Tunnel' -A 1 | tail -n 1 | \
+ sed -n 's/.*| *\(https:\/\/[^ ]*\).*/\1/p')
+
+ echo "$service_url"
+
+[no-cd]
+_print-tunnel-urls fmtm_url api_url odk_url s3_url:
+ @echo
+ @echo "\033[0;33m ############################################### \033[0m"
+ @echo
+ @echo "\033[0;34m FMTM URL: \033[0m"
+ @echo " {{fmtm_url}}"
+ @echo
+ @echo "\033[0;34m API URL: \033[0m"
+ @echo " {{api_url}}"
+ @echo
+ @echo "\033[0;34m ODK Central URL: \033[0m"
+ @echo " {{odk_url}}"
+ @echo
+ @echo "\033[0;34m S3 URL: \033[0m"
+ @echo " {{s3_url}}"
+ @echo
+ @echo "\033[0;33m ############################################### \033[0m"
+ @echo
+
+
+
diff --git a/contrib/tunnel/fmtm/docker-compose.yml b/contrib/just/stop/Justfile
similarity index 64%
rename from contrib/tunnel/fmtm/docker-compose.yml
rename to contrib/just/stop/Justfile
index bdd0aa7eff..8237b78a8b 100644
--- a/contrib/tunnel/fmtm/docker-compose.yml
+++ b/contrib/just/stop/Justfile
@@ -1,4 +1,5 @@
-# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
+#
# This file is part of FMTM.
#
# FMTM is free software: you can redistribute it and/or modify
@@ -15,17 +16,24 @@
# along with FMTM. If not, see .
#
-networks:
- fmtm-net:
- name: fmtm-${GIT_BRANCH:-local}
+# Stop FMTM
+[no-cd]
+default:
+ docker compose down
+
+# Stop FMTM & JOSM
+[no-cd]
+josm:
+ docker compose \
+ -f docker-compose.yml \
+ -f contrib/josm/docker-compose.yml \
+ down
+
+# Stop FMTM & tunnels
+[no-cd]
+tunnel:
+ docker compose \
+ -f docker-compose.yml \
+ -f contrib/tunnel/docker-compose.yml \
+ down
-services:
- fmtm-tunnel:
- image: "docker.io/cloudflare/cloudflared:latest"
- depends_on:
- proxy:
- condition: service_healthy
- networks:
- - fmtm-net
- restart: "unless-stopped"
- command: tunnel --url http://proxy:80
diff --git a/contrib/just/test/Justfile b/contrib/just/test/Justfile
new file mode 100644
index 0000000000..62b6610f25
--- /dev/null
+++ b/contrib/just/test/Justfile
@@ -0,0 +1,37 @@
+# Copyright (c) 2024 Humanitarian OpenStreetMap Team
+#
+# This file is part of FMTM.
+#
+# FMTM is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# FMTM is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with FMTM. If not, see .
+#
+
+# Test backend & frontend
+[no-cd]
+all: backend frontend
+
+# Test backend with pytest
+[no-cd]
+backend:
+ docker compose run --rm api pytest
+
+# Test frontend with Playwright
+[no-cd]
+frontend:
+ docker compose run --rm ui-test
+
+# Check coverage for backend tests
+[no-cd]
+coverage:
+ docker compose run --rm --entrypoint='sh -c' api \
+ 'coverage run -m pytest && coverage report -m'
diff --git a/contrib/tunnel/docker-compose.yml b/contrib/tunnel/docker-compose.yml
new file mode 100644
index 0000000000..5ae8742926
--- /dev/null
+++ b/contrib/tunnel/docker-compose.yml
@@ -0,0 +1,61 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of FMTM.
+#
+# FMTM is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# FMTM is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with FMTM. If not, see .
+#
+
+networks:
+ fmtm-net:
+ name: fmtm-${GIT_BRANCH:-local}
+
+services:
+ frontend-tunnel:
+ image: "docker.io/cloudflare/cloudflared:2024.5.0"
+ depends_on:
+ proxy:
+ condition: service_healthy
+ networks:
+ - fmtm-net
+ restart: "unless-stopped"
+ command: tunnel --url http://proxy:80
+
+ api-tunnel:
+ image: "docker.io/cloudflare/cloudflared:2024.5.0"
+ depends_on:
+ api:
+ condition: service_healthy
+ networks:
+ - fmtm-net
+ restart: "unless-stopped"
+ command: tunnel --url http://api:8000
+
+ central-tunnel:
+ image: "docker.io/cloudflare/cloudflared:2024.5.0"
+ depends_on:
+ central:
+ condition: service_healthy
+ networks:
+ - fmtm-net
+ restart: "unless-stopped"
+ command: tunnel --url http://central:8383
+
+ s3-tunnel:
+ image: "docker.io/cloudflare/cloudflared:2024.5.0"
+ depends_on:
+ s3:
+ condition: service_healthy
+ networks:
+ - fmtm-net
+ restart: "unless-stopped"
+ command: tunnel --url http://s3:9000
diff --git a/docker-compose.yml b/docker-compose.yml
index 6f24d22d1f..e1347aa100 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -119,7 +119,7 @@ services:
- ./src/frontend:/app
- /app/node_modules/
environment:
- - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
+ - VITE_API_URL=${API_URL_OVERRIDE:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}}
ports:
- "7051:7051"
networks:
diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md
index 5adca39bbc..79e0a018bd 100644
--- a/docs/dev/Backend.md
+++ b/docs/dev/Backend.md
@@ -55,6 +55,9 @@ To run the local development setup without ODK Central (use external server):
```bash
dc --profile no-odk up -d
+
+# Or via Just
+just start without-central
```
## 2. Start the API without Docker
@@ -93,6 +96,9 @@ The API should now be accessible at:
```bash
docker compose up -d migrations
+
+# Or via Just
+just migrate
```
### Type Checking
@@ -166,6 +172,9 @@ To run the backend tests locally, run:
```bash
docker compose run --rm api pytest
+
+# Or via Just
+just test backend
```
To assess coverage of tests, run:
@@ -173,6 +182,9 @@ To assess coverage of tests, run:
```bash
docker compose run --rm --entrypoint='sh -c' api \
'coverage run -m pytest && coverage report -m'
+
+# Or via Just
+just test coverage
```
To assess performance of endpoints:
@@ -208,13 +220,6 @@ Creating a new release during development may not always be feasible.
The s3fs tool allows you to mount an S3 bucket on your filesystem,
to browse like any other directory.
-Install:
-
-```bash
-sudo apt update
-sudo apt install s3fs
-```
-
Create a credentials file:
```bash
@@ -223,6 +228,21 @@ echo ACCESS_KEY_ID:SECRET_ACCESS_KEY > ${HOME}/.passwd-s3fs
chmod 600 ${HOME}/.passwd-s3fs
```
+#### Mount local S3 using Just
+
+```bash
+just mount-s3
+```
+
+#### Mount S3 manually
+
+Install s3fs:
+
+```bash
+sudo apt update
+sudo apt install s3fs
+```
+
Mount your bucket:
> If you wish for this to be permanent, see below.
@@ -248,88 +268,39 @@ url=http://s3.fmtm.localhost:7050 0 0`
### Running JOSM in the dev stack
-- Run JOSM with FMTM:
+- Run JOSM with FMTM via Just:
```bash
-docker compose \
- -f docker-compose.yml \
- -f contrib/josm/docker-compose.yml \
- up -d
+just start josm
```
This adds JOSM to the docker compose stack for local development.
-Access the JOSM Remote API:
-Access the JOSM GUI in browser:
You can now call the JOSM API from FMTM and changes will be reflected in the GUI.
-### Debugging local FMTM on mobile
+### Debugging local services on mobile
- It's difficult to debug services running on localhost from your mobile phone.
- An easy way to do this is by tunneling: Cloudflare provides a great free
solution for this (an alternative is Ngrok).
-- To run the tunnel to the FMTM API:
-
- ```bash
- docker compose \
- -f docker-compose.yml \
- -f contrib/tunnel/fmtm/docker-compose.yml \
- up -d
- ```
-
-- View the website to access FMTM remotely (e.g. via mobile):
-
- ```bash
- docker compose \
- -f docker-compose.yml \
- -f contrib/tunnel/fmtm/docker-compose.yml \
- logs fmtm-tunnel
- ```
-
-- Now the final step is to add the provided tunnel URL to the allowed CORS
- origins on API startup:
-
- ```bash
- EXTRA_CORS_ORIGINS=https://the-url-you-were-given.trycloudflare.com \
- docker compose restart api
- ```
-
-### Using local ODK Central on mobile
-
-- Sometimes you wish to use a project in your local ODK Central, via ODK Collect
- on your mobile.
-- To run the tunnel to the ODK Central API:
-
- ```bash
- docker compose \
- -f docker-compose.yml \
- -f contrib/tunnel/odk/docker-compose.yml \
- up -d
- ```
-
-- View the website to access ODK Central remotely (e.g. via mobile):
-
- ```bash
- docker compose \
- -f docker-compose.yml \
- -f contrib/tunnel/odk/docker-compose.yml \
- logs central-tunnel
- ```
-
-1. Requirement: Restart ODK Central using the domain override
- (required for form download URLs):
-
- ```bash
- CENTRAL_DOMAIN_OVERRIDE=the-domain-without-protocol.trycloudflare.com \
- docker compose restart central
- ```
+- We may also wish to debug our local ODK Central instance forms on our mobile ODK
+ Collect.
+- To handle both of these instances set up tunnels for all services with:
+
+```bash
+just start tunnel
+```
+
+To complete this setup, two additional steps must be complete:
+
+- **Requirement 1**: For login to work, use the temporary login.
-2. Requirement: During project creation, set the ODK Central server URL to the
- provided tunnel URL for the ODK Central API.
+- **Requirement 2**: During project creation, set the ODK Central server URL
+ to the provided tunnel URL for the ODK Central API.
- > The credentials for the local ODK Central instance are:
- > Username:
- > Password: Password1234
+ > The credentials for the local ODK Central instance are:
+ > Username:
+ > Password: Password1234
-- Now when you access the project via a QRCode on mobile, the connection to ODK
- Central should work.
+Now when you access the project via a QRCode on mobile, the connection to ODK
+Central should work.