build: fix odk tunnel + working external S3 ODK media storage

contrib/just/start/Justfile

@@ -70,9 +70,9 @@ josm:
   @echo "\033[0;33m ############################################### \033[0m"
   @echo
 
-# Start FMTM with tunnels
+# Externally accessible ODK Central for tests
 [no-cd]
-tunnel:
+odk-tunnel:
   #!/usr/bin/env sh
 
   docker compose \
@@ -85,36 +85,25 @@ tunnel:
   # Wait until services ready without HEALTHCHECK
   sleep 5
 
-  fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
-  api_url=$(just --unstable start _get-tunnel-url 'api')
   odk_url=$(just --unstable start _get-tunnel-url 'central')
-  s3_url=$(just --unstable start _get-tunnel-url 's3')
 
-  just --unstable dotenv update "EXTRA_CORS_ORIGINS" "${fmtm_url}"
-  just --unstable dotenv update "S3_ENDPOINT" "${s3_url}"
-
-  # Restart the containers with env vars set
-  # (API url for frontend, domain for Central form download urls)
+  # Restart Central with DOMAIN var set to new tunnel URL.
+  # Is is required to correctly download forms from Collect
   CENTRAL_DOMAIN_OVERRIDE="$(echo "${odk_url}" | sed 's|^https://||')" \
-  API_URL="$(echo "${api_url}" | sed 's|^https://||')" \
   docker compose \
     -f docker-compose.yml \
     -f contrib/tunnel/docker-compose.yml \
-    up -d api ui central
+    up -d central
 
-  just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"
+  just --unstable start _print-tunnel-url "$odk_url"
 
 # View the URLs for created tunnels
 [no-cd]
-view-tunnel-urls:
+view-tunnel-url:
   #!/usr/bin/env sh
 
-  fmtm_url=$(just --unstable start _get-tunnel-url 'frontend')
-  api_url=$(just --unstable start _get-tunnel-url 'api')
   odk_url=$(just --unstable start _get-tunnel-url 'central')
-  s3_url=$(just --unstable start _get-tunnel-url 's3')
-
-  just --unstable start _print-tunnel-urls "$fmtm_url" "$api_url" "$odk_url" "$s3_url"
+  just --unstable start _print-tunnel-url "$odk_url"
 
 [no-cd]
 _get-tunnel-url service_name:
@@ -130,24 +119,12 @@ _get-tunnel-url service_name:
   echo "$service_url"
 
 [no-cd]
-_print-tunnel-urls fmtm_url api_url odk_url s3_url:
+_print-tunnel-url odk_url:
   @echo
   @echo "\033[0;33m ############################################### \033[0m"
   @echo
-  @echo "\033[0;34m FMTM URL: \033[0m"
-  @echo " {{fmtm_url}}"
-  @echo
-  @echo "\033[0;34m API URL: \033[0m"
-  @echo " {{api_url}}"
-  @echo
   @echo "\033[0;34m ODK Central URL: \033[0m"
   @echo " {{odk_url}}"
   @echo
-  @echo "\033[0;34m S3 URL: \033[0m"
-  @echo " {{s3_url}}"
-  @echo
   @echo "\033[0;33m ############################################### \033[0m"
   @echo
-
-
-

contrib/tunnel/docker-compose.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# Copyright (c) Humanitarian OpenStreetMap Team
 # This file is part of FMTM.
 #
 #     FMTM is free software: you can redistribute it and/or modify
@@ -20,26 +20,6 @@ networks:
     name: fmtm-${GIT_BRANCH:-local}
 
 services:
-  frontend-tunnel:
-    image: "docker.io/cloudflare/cloudflared:2024.10.1"
-    depends_on:
-      proxy:
-        condition: service_healthy
-    networks:
-      - fmtm-net
-    restart: "unless-stopped"
-    command: tunnel --url http://proxy:80
-
-  api-tunnel:
-    image: "docker.io/cloudflare/cloudflared:2024.10.1"
-    depends_on:
-      api:
-        condition: service_healthy
-    networks:
-      - fmtm-net
-    restart: "unless-stopped"
-    command: tunnel --url http://api:8000
-
   central-tunnel:
     image: "docker.io/cloudflare/cloudflared:2024.10.1"
     depends_on:
@@ -49,13 +29,3 @@ services:
       - fmtm-net
     restart: "unless-stopped"
     command: tunnel --url http://central:8383
-
-  s3-tunnel:
-    image: "docker.io/cloudflare/cloudflared:2024.10.1"
-    depends_on:
-      s3:
-        condition: service_healthy
-    networks:
-      - fmtm-net
-    restart: "unless-stopped"
-    command: tunnel --url http://s3:9000

docker-compose.development.yml

@@ -158,8 +158,8 @@ services:
       - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137}
       - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee}
       - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632}
-      - S3_ENDPOINT=${S3_ENDPOINT}
-      - S3_ODK_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"}
+      - S3_SERVER=${S3_ENDPOINT}
+      - S3_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"}
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
       - S3_SECRET_KEY=${S3_SECRET_KEY}
     networks:

docker-compose.yml

@@ -131,7 +131,7 @@ services:
       - ./src/frontend:/app
       - /app/node_modules/
     environment:
-      - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}}
+      - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
     ports:
       - "7051:7051"
     networks:
@@ -154,7 +154,7 @@ services:
       - /app/.svelte-kit/
       # - ../ui:/app/node_modules/@hotosm/ui:ro
     environment:
-      - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}}
+      - VITE_API_URL=http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
       - VITE_SYNC_URL=http://sync.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
     networks:
       - fmtm-net
@@ -194,8 +194,9 @@ services:
       - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137}
       - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee}
       - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632}
-      - S3_ENDPOINT=${S3_ENDPOINT:-"http://s3:9000"}
-      - S3_ODK_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"}
+      # Note S3_ENDPOINT is hardcoded here for when we use tunnel config
+      - S3_SERVER="http://s3:9000
+      - S3_BUCKET_NAME=${S3_ODK_BUCKET_NAME:-"fmtm-odk-media"}
       - S3_ACCESS_KEY=${S3_ACCESS_KEY}
       - S3_SECRET_KEY=${S3_SECRET_KEY}
     # ports:
@@ -326,7 +327,8 @@ services:
       - .env
     # Hardcode some vars for dev, as not necessarily present in the .env file
     environment:
-      - S3_ENDPOINT=${S3_ENDPOINT:-"http://s3:9000"}
+      # Note S3_ENDPOINT is hardcoded here for when we use tunnel config
+      - S3_ENDPOINT="http://s3:9000
       - S3_BACKUP_BUCKET_NAME=${S3_BACKUP_BUCKET_NAME:-"fmtm-db-backups"}
     networks:
       - fmtm-net

docs/dev/Backend.md

@@ -277,29 +277,20 @@ This adds JOSM to the docker compose stack for local development.
 
 You can now call the JOSM API from FMTM and changes will be reflected in the GUI.
 
-### Debugging local services on mobile
+### Debugging ODK forms when running on localhost
 
-- It's difficult to debug services running on localhost from your mobile phone.
-- An easy way to do this is by tunneling: Cloudflare provides a great free
-  solution for this (an alternative is Ngrok).
-- We may also wish to debug our local ODK Central instance forms on our mobile ODK
-  Collect.
-- To handle both of these instances set up tunnels for all services with:
+- ODK Collect requires an externally accessible instance of Central.
+- To achieve this for local development / debugging, a good solution is Cloudflare
+  tunnelling (alternative to Ngrok).
+- There is a helper script to do this automatically for you:
 
-```bash
-just start tunnel
-```
-
-To complete this setup, two additional steps must be complete:
-
-- **Requirement 1**: For login to work, use the temporary login.
-
-- **Requirement 2**: During project creation, set the ODK Central server URL
-  to the provided tunnel URL for the ODK Central API.
+  ```bash
+  just start odk-tunnel
+  ```
 
-  > The credentials for the local ODK Central instance are:
-  > Username: <[email protected]>
-  > Password: Password1234
+Once started, use the output ODK Central URL from the terminal during
+project creation. The QRCode should now work in ODK Collect.
 
-Now when you access the project via a QRCode on mobile, the connection to ODK
-Central should work.
+> The credentials for the local ODK Central instance are:
+> Username: <[email protected]>
+> Password: Password1234

odkcentral/api/container-entrypoint.sh

@@ -3,8 +3,8 @@
 set -eo pipefail
 
 check_all_s3_vars_present() {
-    if [ -z "${S3_ENDPOINT}" ]; then
-        echo "Environment variable S3_ENDPOINT is not set."
+    if [ -z "${S3_SERVER}" ]; then
+        echo "Environment variable S3_SERVER is not set."
         exit 1
     fi
     if [ -z "${S3_ACCESS_KEY}" ]; then
@@ -15,16 +15,16 @@ check_all_s3_vars_present() {
         echo "Environment variable S3_SECRET_KEY is not set."
         exit 1
     fi
-    if [ -z "${S3_ODK_BUCKET_NAME}" ]; then
-        echo "Environment variable S3_ODK_BUCKET_NAME is not set."
+    if [ -z "${S3_BUCKET_NAME}" ]; then
+        echo "Environment variable S3_BUCKET_NAME is not set."
         exit 1
     fi
 
     # Strip any extra unrequired "quotes"
-    export S3_ENDPOINT="${S3_ENDPOINT//\"/}"
+    export S3_SERVER="${S3_SERVER//\"/}"
     export S3_ACCESS_KEY="${S3_ACCESS_KEY//\"/}"
     export S3_SECRET_KEY="${S3_SECRET_KEY//\"/}"
-    export S3_ODK_BUCKET_NAME="${S3_ODK_BUCKET_NAME//\"/}"
+    export S3_BUCKET_NAME="${S3_BUCKET_NAME//\"/}"
 }
 
 # Check env vars + strip extra quotes on vars
@@ -50,11 +50,11 @@ echo "Elevating user to admin"
 odk-cmd --email "${SYSADMIN_EMAIL}" user-promote || true
 
 ### Create S3 bucket for submission photo storage ###
-BUCKET_NAME="${S3_ODK_BUCKET_NAME}"
-echo "Creating S3 bucket ${BUCKET_NAME} to store submission media"
-mc alias set s3 "$S3_ENDPOINT" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
-mc mb "s3/${BUCKET_NAME}" --ignore-existing
-mc anonymous set download "s3/${BUCKET_NAME}"
+echo "Creating S3 bucket ${S3_BUCKET_NAME} to store submission media"
+mc alias set s3 "$S3_SERVER" "$S3_ACCESS_KEY" "$S3_SECRET_KEY"
+mc mb "s3/${S3_BUCKET_NAME}" --ignore-existing
+# Prevent anonymous access (pre-signed URL download only)
+mc anonymous set none "s3/${S3_BUCKET_NAME}"
 
 ### Run server (hardcode WORKER_COUNT=1 for dev) ###
 export WORKER_COUNT=1