Skip to content

Commit

Permalink
build: update Just and Justfile modules for better dev experience (#1537
Browse files Browse the repository at this point in the history
)

* build: update all just commands, working just start tunnel

* docs: update backend info using just commands
  • Loading branch information
spwoodcock authored May 24, 2024
1 parent dfb5e7b commit 5bf93cb
Show file tree
Hide file tree
Showing 10 changed files with 392 additions and 182 deletions.
123 changes: 50 additions & 73 deletions Justfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Copyright (c) 2020, 2021 Humanitarian OpenStreetMap Team
# Copyright (c) 2024 Humanitarian OpenStreetMap Team
#
# This file is part of FMTM.
#
Expand All @@ -16,62 +16,37 @@
# along with FMTM. If not, see <https:#www.gnu.org/licenses/>.
#

# Builds
mod start 'contrib/just/start/Justfile'
mod stop 'contrib/just/stop/Justfile'
mod build 'contrib/just/build/Justfile'
mod test 'contrib/just/test/Justfile'
mod dotenv 'contrib/just/dotenv/Justfile'

build-backend:
docker compose build api
# Run the help script
default:
@just --unstable help

build-frontend:
docker compose build ui
# View available commands
help:
@just --unstable --list --justfile {{justfile()}}

build: build-backend build-frontend
# Run database migrations for backend
migrate:
docker compose up -d migrations

# Run

run:
docker compose up -d

run-without-central:
docker compose --profile no-odk up -d

run-with-josm:
docker compose \
-f docker-compose.yml \
-f contrib/josm/docker-compose.yml \
up -d

run-with-tunnels:
docker compose \
-f docker-compose.yml \
-f contrib/tunnel/fmtm/docker-compose.yml \
-f contrib/tunnel/odk/docker-compose.yml \
up -d

stop:
docker compose down

clean-db:
# Delete local database, S3, and ODK Central data
clean:
docker compose down -v

# Tests

test-backend:
docker compose run --rm api pytest

test-frontend:
docker compose run --rm ui-test

test: test-backend test-frontend

# Maintenance

# Run pre-commit hooks
lint:
TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
--volume $PWD:$PWD --workdir $PWD \
--entrypoint='sh -c' api \
'git config --global --add safe.directory $PWD \
&& pre-commit run --all-files'

# Increment version
bump:
TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
--volume $PWD:$PWD --workdir $PWD \
Expand All @@ -82,33 +57,35 @@ bump:
&& cd src/backend \
&& cz bump --check-consistency'

# Docs

docs-rebuild: docs-clean docs-doxygen docs-uml

docs-clean:
@rm -rf docs/{apidocs,html,docbook,man} docs/packages.png docs/classes.png

docs-doxygen:
cd docs && doxygen
# Run docs website locally
docs:
@echo
@echo "\033[0;33m ############################################### \033[0m"
@echo
@echo
@echo "\033[0;34m Access the docs site on: http://localhost:55425 \033[0m"
@echo
@echo
@echo "\033[0;33m ############################################### \033[0m"
@echo

docs-uml:
cd docs && pyreverse -o png ../src/backend/app

docs-pdf:
# Strip any unicode out of the markdown file before converting to PDF
# FIXME
MDS := \
docs/dev/Backend.md \
docs/dev/Database-Tips.md \
docs/dev/Release-Cycle.md \
docs/dev/Frontend.md \
docs/dev/Production.md \
docs/dev/Version-Control.md \
docs/dev/Setup.md \
docs/dev/Troubleshooting.md \
PDFS := $(MDS:.md=.pdf)
@echo "Converting $PDFS to a PDF"
@new=$(notdir $(basename $PDFS)); \
iconv -f utf-8 -t US $PDFS -c | \
pandoc $PDFS -f markdown -t pdf -s -o /tmp/$$new.pdf
TAG_OVERRIDE=ci TARGET_OVERRIDE=ci docker compose run --rm --no-deps \
--volume $PWD:$PWD --workdir $PWD --publish 55425:3000 \
--entrypoint='sh -c' api \
'git config --global --add safe.directory $PWD \
&& mkdocs serve --dev-addr 0.0.0.0:3000'

# Mount an S3 bucket on your filesystem
mount-s3:
#!/usr/bin/env sh
fstab_entry="fmtm-data /mnt/fmtm/local fuse.s3fs _netdev,allow_other,\
use_path_request_style,passwd_file=/home/$(whoami)/s3-creds/fmtm-local,\
url=http://s3.fmtm.localhost:7050 0 0"
if ! grep -q "$fstab_entry" /etc/fstab; then
echo "Mounting local FMTM S3 permanently in /etc/fstab"
echo "$fstab_entry" | sudo tee -a /etc/fstab > /dev/null
echo
else
echo "Local FMTM S3 is already mounted"
fi
6 changes: 3 additions & 3 deletions contrib/josm/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ services:
josm:
image: "ghcr.io/hotosm/fmtm/josm:latest"
build:
context: .
context: contrib/josm
container_name: josm
environment:
- DISPLAY=josm-novnc:0.0
Expand All @@ -32,13 +32,13 @@ services:
- fmtm-net
- x11
ports:
- 8111:80
- "8111:80"
restart: "unless-stopped"

josm-novnc:
image: "ghcr.io/hotosm/fmtm/josm-novnc:latest"
build:
context: novnc
context: contrib/josm/novnc
container_name: josm_novnc
environment:
- DISPLAY_WIDTH=1280
Expand Down
13 changes: 13 additions & 0 deletions contrib/just/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Just Submodules

- This directory contains submodules for the parent Justfile.
- It allows for submodules to be used like:

```bash
just build frontend
just build backend

# Instead of (only top level)
just build-backend
just build-frontend
```
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
# Copyright (c) 2024 Humanitarian OpenStreetMap Team
#
# This file is part of FMTM.
#
# FMTM is free software: you can redistribute it and/or modify
Expand All @@ -15,17 +16,19 @@
# along with FMTM. If not, see <https:#www.gnu.org/licenses/>.
#

networks:
fmtm-net:
name: fmtm-${GIT_BRANCH:-local}
# Update a variable in the .env file
[no-cd]
update key value:
#!/usr/bin/env sh
var_name={{key}}
var_value={{value}}
var_pattern="^${var_name}="
new_var="${var_name}=${var_value}"

if grep -Eq "${var_pattern}" .env; then
sed -i "s|${var_pattern}.*|${new_var}|" .env
else
echo "${new_var}" >> .env
fi

services:
central-tunnel:
image: "docker.io/cloudflare/cloudflared:latest"
depends_on:
central:
condition: service_healthy
networks:
- fmtm-net
restart: "unless-stopped"
command: tunnel --url http://central:8383
140 changes: 140 additions & 0 deletions contrib/just/start/Justfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
# Copyright (c) 2024 Humanitarian OpenStreetMap Team
#
# This file is part of FMTM.
#
# FMTM is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# FMTM is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with FMTM. If not, see <https:#www.gnu.org/licenses/>.
#

# Start FMTM
[no-cd]
default:
docker compose up -d

# Start backend API only
[no-cd]
backend:
docker compose up -d api

# Start frontend UI only
[no-cd]
frontend:
docker compose up -d ui

# Start FMTM without ODK Central
[no-cd]
without-central:
docker compose --profile no-odk up -d

# Start FMTM with JOSM
[no-cd]
josm:
docker compose \
-f docker-compose.yml \
-f contrib/josm/docker-compose.yml \
up -d

@echo
@echo "\033[0;33m ############################################### \033[0m"
@echo
@echo " Access the JOSM Remote API: http://localhost:8111"
@echo " Access the JOSM GUI in browser: http://localhost:8112"
@echo
@echo "\033[0;33m ############################################### \033[0m"
@echo

# Start FMTM with tunnels
[no-cd]
tunnel:
#!/usr/bin/env sh
docker compose \
-f docker-compose.yml \
-f contrib/tunnel/docker-compose.yml \
up -d --wait

# Workaround to until PR merged:
# https://github.com/cloudflare/cloudflared/pull/1135
# Wait until services ready without HEALTHCHECK
sleep 5

fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
api_url=$(just --unstable start _get-tunnel-url 'api')
odk_url=$(just --unstable start _get-tunnel-url 'central')
s3_url=$(just --unstable start _get-tunnel-url 's3')

just --unstable dotenv update "EXTRA_CORS_ORIGINS" "${fmtm_url}"
just --unstable dotenv update "S3_ENDPOINT" "${s3_url}"

# Restart the API and UI with environment variables set
API_URL_OVERRIDE="${api_url}" docker compose \
-f docker-compose.yml \
-f contrib/tunnel/docker-compose.yml \
up -d api ui

# Restart ODK Central with domain override (for form download urls)
CENTRAL_DOMAIN_OVERRIDE="$(echo "${odk_url}" | sed 's|^https://||')" \
docker compose \
-f docker-compose.yml \
-f contrib/tunnel/docker-compose.yml \
up -d central

just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"

# View the URLs for created tunnels
[no-cd]
view-tunnel-urls:
#!/usr/bin/env sh
fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
api_url=$(just --unstable start _get-tunnel-url 'api')
odk_url=$(just --unstable start _get-tunnel-url 'central')
s3_url=$(just --unstable start _get-tunnel-url 's3')

just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"

[no-cd]
_get-tunnel-url service_name:
#!/usr/bin/env sh
service_url=$(docker compose \
-f docker-compose.yml \
-f contrib/tunnel/docker-compose.yml \
logs {{service_name}}-tunnel | \
grep 'Your quick Tunnel' -A 1 | tail -n 1 | \
sed -n 's/.*| *\(https:\/\/[^ ]*\).*/\1/p')

echo "$service_url"

[no-cd]
_print-tunnel-urls fmtm_url api_url odk_url s3_url:
@echo
@echo "\033[0;33m ############################################### \033[0m"
@echo
@echo "\033[0;34m FMTM URL: \033[0m"
@echo " {{fmtm_url}}"
@echo
@echo "\033[0;34m API URL: \033[0m"
@echo " {{api_url}}"
@echo
@echo "\033[0;34m ODK Central URL: \033[0m"
@echo " {{odk_url}}"
@echo
@echo "\033[0;34m S3 URL: \033[0m"
@echo " {{s3_url}}"
@echo
@echo "\033[0;33m ############################################### \033[0m"
@echo



Loading

0 comments on commit 5bf93cb

Please sign in to comment.