diff --git a/contrib/just/start/Justfile b/contrib/just/start/Justfile index f84169bb1d..4e0c13cbb7 100644 --- a/contrib/just/start/Justfile +++ b/contrib/just/start/Justfile @@ -70,9 +70,9 @@ josm: @echo "\033[0;33m ############################################### \033[0m" @echo -# Start FMTM with tunnels +# Externally accessible ODK Central for tests [no-cd] -tunnel: +odk-tunnel: #!/usr/bin/env sh docker compose \ @@ -85,36 +85,25 @@ tunnel: # Wait until services ready without HEALTHCHECK sleep 5 - fmtm_url=$(just --unstable start _get-tunnel-url 'frontend') - api_url=$(just --unstable start _get-tunnel-url 'api') odk_url=$(just --unstable start _get-tunnel-url 'central') - s3_url=$(just --unstable start _get-tunnel-url 's3') - just --unstable dotenv update "EXTRA_CORS_ORIGINS" "${fmtm_url}" - just --unstable dotenv update "S3_ENDPOINT" "${s3_url}" - - # Restart the containers with env vars set - # (API url for frontend, domain for Central form download urls) + # Restart Central with DOMAIN var set to new tunnel URL. + # Is is required to correctly download forms from Collect CENTRAL_DOMAIN_OVERRIDE="$(echo "${odk_url}" | sed 's|^https://||')" \ - API_URL="$(echo "${api_url}" | sed 's|^https://||')" \ docker compose \ -f docker-compose.yml \ -f contrib/tunnel/docker-compose.yml \ - up -d api ui central + up -d central - just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url" + just --unstable start _print-tunnel-url "$odk_url" # View the URLs for created tunnels [no-cd] -view-tunnel-urls: +view-tunnel-url: #!/usr/bin/env sh - fmtm_url=$(just --unstable start _get-tunnel-url 'frontend') - api_url=$(just --unstable start _get-tunnel-url 'api') odk_url=$(just --unstable start _get-tunnel-url 'central') - s3_url=$(just --unstable start _get-tunnel-url 's3') - - just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url" + just --unstable start _print-tunnel-url "$odk_url" [no-cd] _get-tunnel-url service_name: @@ -130,24 +119,12 @@ _get-tunnel-url service_name: echo "$service_url" [no-cd] -_print-tunnel-urls fmtm_url api_url odk_url s3_url: +_print-tunnel-url odk_url: @echo @echo "\033[0;33m ############################################### \033[0m" @echo - @echo "\033[0;34m FMTM URL: \033[0m" - @echo " {{fmtm_url}}" - @echo - @echo "\033[0;34m API URL: \033[0m" - @echo " {{api_url}}" - @echo @echo "\033[0;34m ODK Central URL: \033[0m" @echo " {{odk_url}}" @echo - @echo "\033[0;34m S3 URL: \033[0m" - @echo " {{s3_url}}" - @echo @echo "\033[0;33m ############################################### \033[0m" @echo - - - diff --git a/contrib/tunnel/docker-compose.yml b/contrib/tunnel/docker-compose.yml index 3b29b9fb0d..192a5e979b 100644 --- a/contrib/tunnel/docker-compose.yml +++ b/contrib/tunnel/docker-compose.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team +# Copyright (c) Humanitarian OpenStreetMap Team # This file is part of FMTM. # # FMTM is free software: you can redistribute it and/or modify @@ -20,26 +20,6 @@ networks: name: fmtm-${GIT_BRANCH:-local} services: - frontend-tunnel: - image: "docker.io/cloudflare/cloudflared:2024.10.1" - depends_on: - proxy: - condition: service_healthy - networks: - - fmtm-net - restart: "unless-stopped" - command: tunnel --url http://proxy:80 - - api-tunnel: - image: "docker.io/cloudflare/cloudflared:2024.10.1" - depends_on: - api: - condition: service_healthy - networks: - - fmtm-net - restart: "unless-stopped" - command: tunnel --url http://api:8000 - central-tunnel: image: "docker.io/cloudflare/cloudflared:2024.10.1" depends_on: @@ -49,13 +29,3 @@ services: - fmtm-net restart: "unless-stopped" command: tunnel --url http://central:8383 - - s3-tunnel: - image: "docker.io/cloudflare/cloudflared:2024.10.1" - depends_on: - s3: - condition: service_healthy - networks: - - fmtm-net - restart: "unless-stopped" - command: tunnel --url http://s3:9000 diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 9c3db89075..019c4586d8 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -158,8 +158,8 @@ services: - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} - - S3_ENDPOINT=${S3_ENDPOINT} - - S3_ODK_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"} + - S3_SERVER=${S3_ENDPOINT} + - S3_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"} - S3_ACCESS_KEY=${S3_ACCESS_KEY} - S3_SECRET_KEY=${S3_SECRET_KEY} networks: diff --git a/docker-compose.yml b/docker-compose.yml index 162043d3e1..15314a8ebc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,7 +131,7 @@ services: - ./src/frontend:/app - /app/node_modules/ environment: - - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}} + - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} ports: - "7051:7051" networks: @@ -154,7 +154,7 @@ services: - /app/.svelte-kit/ # - ../ui:/app/node_modules/@hotosm/ui:ro environment: - - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}} + - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} - VITE_SYNC_URL=http://sync.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} networks: - fmtm-net @@ -194,8 +194,9 @@ services: - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} - - S3_ENDPOINT=${S3_ENDPOINT:-"http://s3:9000"} - - S3_ODK_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"} + # Note S3_ENDPOINT is hardcoded here for when we use tunnel config + - S3_SERVER="http://s3:9000 + - S3_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"} - S3_ACCESS_KEY=${S3_ACCESS_KEY} - S3_SECRET_KEY=${S3_SECRET_KEY} # ports: @@ -326,7 +327,8 @@ services: - .env # Hardcode some vars for dev, as not necessarily present in the .env file environment: - - S3_ENDPOINT=${S3_ENDPOINT:-"http://s3:9000"} + # Note S3_ENDPOINT is hardcoded here for when we use tunnel config + - S3_ENDPOINT="http://s3:9000 - S3_BACKUP_BUCKET_NAME=${S3_BACKUP_BUCKET_NAME:-"fmtm-db-backups"} networks: - fmtm-net diff --git a/docs/dev/Backend.md b/docs/dev/Backend.md index c15b719d8e..f09101c703 100644 --- a/docs/dev/Backend.md +++ b/docs/dev/Backend.md @@ -277,29 +277,20 @@ This adds JOSM to the docker compose stack for local development. You can now call the JOSM API from FMTM and changes will be reflected in the GUI. -### Debugging local services on mobile +### Debugging ODK forms when running on localhost -- It's difficult to debug services running on localhost from your mobile phone. -- An easy way to do this is by tunneling: Cloudflare provides a great free - solution for this (an alternative is Ngrok). -- We may also wish to debug our local ODK Central instance forms on our mobile ODK - Collect. -- To handle both of these instances set up tunnels for all services with: +- ODK Collect requires an externally accessible instance of Central. +- To achieve this for local development / debugging, a good solution is Cloudflare + tunnelling (alternative to Ngrok). +- There is a helper script to do this automatically for you: -```bash -just start tunnel -``` - -To complete this setup, two additional steps must be complete: - -- **Requirement 1**: For login to work, use the temporary login. - -- **Requirement 2**: During project creation, set the ODK Central server URL - to the provided tunnel URL for the ODK Central API. + ```bash + just start odk-tunnel + ``` - > The credentials for the local ODK Central instance are: - > Username: - > Password: Password1234 +Once started, use the output ODK Central URL from the terminal during +project creation. The QRCode should now work in ODK Collect. -Now when you access the project via a QRCode on mobile, the connection to ODK -Central should work. +> The credentials for the local ODK Central instance are: +> Username: +> Password: Password1234 diff --git a/odkcentral/api/container-entrypoint.sh b/odkcentral/api/container-entrypoint.sh index c2c0eaa256..5d44ff2250 100644 --- a/odkcentral/api/container-entrypoint.sh +++ b/odkcentral/api/container-entrypoint.sh @@ -3,8 +3,8 @@ set -eo pipefail check_all_s3_vars_present() { - if [ -z "${S3_ENDPOINT}" ]; then - echo "Environment variable S3_ENDPOINT is not set." + if [ -z "${S3_SERVER}" ]; then + echo "Environment variable S3_SERVER is not set." exit 1 fi if [ -z "${S3_ACCESS_KEY}" ]; then @@ -15,16 +15,16 @@ check_all_s3_vars_present() { echo "Environment variable S3_SECRET_KEY is not set." exit 1 fi - if [ -z "${S3_ODK_BUCKET_NAME}" ]; then - echo "Environment variable S3_ODK_BUCKET_NAME is not set." + if [ -z "${S3_BUCKET_NAME}" ]; then + echo "Environment variable S3_BUCKET_NAME is not set." exit 1 fi # Strip any extra unrequired "quotes" - export S3_ENDPOINT="${S3_ENDPOINT//\"/}" + export S3_SERVER="${S3_SERVER//\"/}" export S3_ACCESS_KEY="${S3_ACCESS_KEY//\"/}" export S3_SECRET_KEY="${S3_SECRET_KEY//\"/}" - export S3_ODK_BUCKET_NAME="${S3_ODK_BUCKET_NAME//\"/}" + export S3_BUCKET_NAME="${S3_BUCKET_NAME//\"/}" } # Check env vars + strip extra quotes on vars @@ -50,11 +50,11 @@ echo "Elevating user to admin" odk-cmd --email "${SYSADMIN_EMAIL}" user-promote || true ### Create S3 bucket for submission photo storage ### -BUCKET_NAME="${S3_ODK_BUCKET_NAME}" -echo "Creating S3 bucket ${BUCKET_NAME} to store submission media" -mc alias set s3 "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" -mc mb "s3/${BUCKET_NAME}" --ignore-existing -mc anonymous set download "s3/${BUCKET_NAME}" +echo "Creating S3 bucket ${S3_BUCKET_NAME} to store submission media" +mc alias set s3 "$S3_SERVER" "$S3_ACCESS_KEY" "$S3_SECRET_KEY" +mc mb "s3/${S3_BUCKET_NAME}" --ignore-existing +# Prevent anonymous access (pre-signed URL download only) +mc anonymous set none "s3/${S3_BUCKET_NAME}" ### Run server (hardcode WORKER_COUNT=1 for dev) ### export WORKER_COUNT=1