build: add config for running electric behind proxy in production

docker-compose.development.yml

@@ -59,6 +59,8 @@ services:
         condition: service_completed_successfully
       ui:
         condition: service_completed_successfully
+      electric:
+        condition: service_started
     volumes:
       - fmtm_frontend:/usr/share/nginx/html/fmtm
       - central_frontend:/usr/share/nginx/html/central/
@@ -69,6 +71,7 @@ services:
       FMTM_API_DOMAIN: ${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}}
       FMTM_ODK_DOMAIN: ${FMTM_ODK_DOMAIN:-odk.${FMTM_DOMAIN}}
       FMTM_S3_DOMAIN: ${FMTM_S3_DOMAIN:-s3.${FMTM_DOMAIN}}
+      FMTM_SYNC_DOMAIN: ${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}}
     ports:
       - 80:80
       - 443:443
@@ -119,6 +122,7 @@ services:
       args:
         APP_VERSION: ${GIT_BRANCH}
         VITE_API_URL: https://${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}}
+        VITE_SYNC_URL: https://${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}}
         NODE_ENV: development
     volumes:
       - fmtm_frontend:/frontend
@@ -216,6 +220,22 @@ services:
       timeout: 5s
       retries: 3
 
+  electric:
+    image: "electricsql/electric:${ELECTRIC_TAG:-0.7.7}"
+    depends_on:
+      fmtm-db:
+        condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
+    environment:
+      DATABASE_URL: postgresql://${FMTM_DB_USER}:${FMTM_DB_PASSWORD}@${FMTM_DB_HOST:-fmtm-db}/${FMTM_DB_NAME:-fmtm}?sslmode=disable
+      # OTEL_EXPORT: otlp
+      # OTLP_ENDPOINT: https://...
+      # ELECTRIC_WRITE_TO_PG_MODE: direct_writes
+    networks:
+      - fmtm-net
+    restart: "unless-stopped"
+
   central-db:
     image: "postgis/postgis:${POSTGIS_TAG:-14-3.5-alpine}"
     volumes:

docker-compose.main.yml

@@ -51,6 +51,8 @@ services:
         condition: service_started
       ui:
         condition: service_completed_successfully
+      electric:
+        condition: service_started
     volumes:
       - fmtm_frontend:/usr/share/nginx/html/fmtm
       - certs:/etc/letsencrypt
@@ -59,6 +61,7 @@ services:
       FMTM_DOMAIN: ${FMTM_DOMAIN}
       FMTM_API_DOMAIN: ${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}}
       FMTM_S3_DOMAIN: ${FMTM_S3_DOMAIN:-s3.${FMTM_DOMAIN}}
+      FMTM_SYNC_DOMAIN: ${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}}
     ports:
       - 80:80
       - 443:443
@@ -109,6 +112,7 @@ services:
       args:
         APP_VERSION: main
         VITE_API_URL: https://${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}}
+        VITE_SYNC_URL: https://${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}}
         NODE_ENV: production
     volumes:
       - fmtm_frontend:/frontend
@@ -156,6 +160,22 @@ services:
       timeout: 5s
       retries: 3
 
+  electric:
+    image: "electricsql/electric:${ELECTRIC_TAG:-0.7.7}"
+    depends_on:
+      fmtm-db:
+        condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
+    environment:
+      DATABASE_URL: postgresql://${FMTM_DB_USER}:${FMTM_DB_PASSWORD}@${FMTM_DB_HOST:-fmtm-db}/${FMTM_DB_NAME:-fmtm}?sslmode=disable
+      # OTEL_EXPORT: otlp
+      # OTLP_ENDPOINT: https://...
+      # ELECTRIC_WRITE_TO_PG_MODE: direct_writes
+    networks:
+      - fmtm-net
+    restart: "unless-stopped"
+
   migrations:
     image: "ghcr.io/hotosm/fmtm/backend:main"
     depends_on:

docker-compose.yml

@@ -53,6 +53,8 @@ services:
         required: false
       s3:
         condition: service_started
+      electric:
+        condition: service_started
     volumes:
       - central_frontend:/usr/share/nginx/html/central
     ports:
@@ -153,6 +155,7 @@ services:
       # - ../ui:/app/node_modules/@hotosm/ui:ro
     environment:
       - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}}
+      - VITE_SYNC_URL=http://sync.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}
     networks:
       - fmtm-net
     restart: "unless-stopped"
@@ -280,13 +283,7 @@ services:
       DATABASE_URL: postgresql://${FMTM_DB_USER:-fmtm}:${FMTM_DB_PASSWORD:-fmtm}@${FMTM_DB_HOST:-fmtm-db}/${FMTM_DB_NAME:-fmtm}?sslmode=disable
       # OTEL_EXPORT: otlp
       # OTLP_ENDPOINT: https://...
-
       # ELECTRIC_WRITE_TO_PG_MODE: direct_writes
-      # AUTH_JWT_ALG: HS384
-      # AUTH_JWT_KEY: ${ENCRYPTION_KEY}
-      # AUTH_JWT_AUD: ${FMTM_DOMAIN}
-    ports:
-      - "7055:3000"
     networks:
       - fmtm-net
     restart: "unless-stopped"

docs/dev/Production.md

@@ -77,6 +77,7 @@ These defaults can be overridden with respective environment variables:
 FMTM_API_DOMAIN
 FMTM_ODK_DOMAIN
 FMTM_S3_DOMAIN
+FMTM_SYNC_DOMAIN
 ```
 
 ### Connecting to a remote database

nginx/Dockerfile

@@ -92,6 +92,7 @@ COPY templates/dev/fmtm.conf.template \
      templates/dev/api.conf.template \
      templates/dev/odk.conf.template \
      templates/dev/minio.conf.template \
+     templates/dev/sync.conf.template \
      /etc/nginx/templates/
 
 
@@ -103,6 +104,7 @@ RUN chmod +x /docker-entrypoint.sh
 COPY templates/cert-init/fmtm.conf.template \
      templates/cert-init/api.conf.template \
      templates/cert-init/minio.conf.template \
+     templates/cert-init/sync.conf.template \
      /etc/nginx/templates/
 
 
@@ -118,6 +120,7 @@ FROM base as main
 COPY templates/fmtm.conf.template \
      templates/api.conf.template \
      templates/minio.conf.template \
+     templates/sync.conf.template \
      /etc/nginx/templates/
 
 

nginx/container-entrypoint.sh

@@ -62,6 +62,12 @@ if [ -n "${FMTM_S3_DOMAIN}" ]; then
     certbot_args+=("-d" "${FMTM_S3_DOMAIN}")
 fi
 
+# Add FMTM_SYNC_DOMAIN if present
+if [ -n "${FMTM_SYNC_DOMAIN}" ]; then
+    echo "Adding ${FMTM_SYNC_DOMAIN} to certificate for domain ${FMTM_DOMAIN}."
+    certbot_args+=("-d" "${FMTM_SYNC_DOMAIN}")
+fi
+
 # Run certbot with the constructed arguments
 echo "Running command: certbot --non-interactive certonly ${certbot_args[*]}"
 certbot --non-interactive certonly "${certbot_args[@]}"

nginx/templates/cert-init/sync.conf.template

@@ -0,0 +1,26 @@
+# Copyright (c) Humanitarian OpenStreetMap Team
+#
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+server {
+    listen 80;
+    server_name ${FMTM_SYNC_DOMAIN};
+
+    location /.well-known/acme-challenge/ {
+        root /var/www/certbot;
+    }
+}

nginx/templates/dev/sync.conf.template

@@ -0,0 +1,62 @@
+# Copyright (c) Humanitarian OpenStreetMap Team
+#
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+upstream sync {
+    server electric:3000 max_fails=1 fail_timeout=2s;
+    keepalive 32;
+}
+
+server {
+    # Default handler for port 80
+    listen 80;
+    server_name sync.fmtm.localhost;
+
+    # Max upload size 10MB
+    client_max_body_size 10M;
+
+    location / {
+        # Max time to initiate connection with electric
+        proxy_connect_timeout 10s;
+        # Max time for a backend response to return
+        proxy_read_timeout 20s;
+        # Max time to send request to backend, i.e. upload
+        proxy_send_timeout 20s;
+
+        # Requests headers
+        proxy_set_header Host                $http_host;
+        proxy_set_header X-Real-IP           $remote_addr;
+        proxy_set_header X-Forwarded-Proto   $scheme;
+        proxy_set_header X-Forwarded-For     $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host    $http_host;
+        proxy_set_header X-Forwarded-Server  $http_host;
+        proxy_set_header X-Forwarded-Port    $server_port;
+
+        # Disable buffer to temp files, tweak buffer for memory
+        proxy_max_temp_file_size 0;
+        proxy_buffer_size 64k;
+        proxy_buffers 8 64k;
+        proxy_busy_buffers_size 64k;
+
+        proxy_pass http://sync;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}

nginx/templates/sync.conf.template

@@ -0,0 +1,79 @@
+# Copyright (c) Humanitarian OpenStreetMap Team
+#
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+upstream sync {
+    server electric:3000 max_fails=1 fail_timeout=2s;
+    keepalive 32;
+}
+
+server {
+    # Default handler for port 80
+    listen 80;
+    server_name ${FMTM_SYNC_DOMAIN};
+    return 301 https://$host$request_uri;
+}
+
+server {
+    # Default handler for port 443
+    listen 443 ssl reuseport;
+    server_name ${FMTM_SYNC_DOMAIN};
+
+    ssl_certificate /etc/letsencrypt/live/${FMTM_DOMAIN}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/${FMTM_DOMAIN}/privkey.pem;
+    include /etc/nginx/options-ssl-nginx.conf;
+    include /etc/nginx/options-security.conf;
+
+    # Max upload size 1GB
+    client_max_body_size 1G;
+
+    # Response headers (Access-Control-Allow-Origin set by FastAPI, not required)
+    add_header 'Content-Security-Policy' 'upgrade-insecure-requests';
+    # For opentelemetry
+    add_header 'Access-Control-Allow-Headers' 'traceparent,tracestate';
+
+    location / {
+        # Max time to initiate connection with electric
+        proxy_connect_timeout 10s;
+        # Max time for a backend response to return
+        proxy_read_timeout 20s;
+        # Max time to send request to backend, i.e. upload
+        proxy_send_timeout 20s;
+
+        # Requests headers
+        proxy_set_header Host                $http_host;
+        proxy_set_header X-Real-IP           $remote_addr;
+        proxy_set_header X-Forwarded-Proto   $scheme;
+        proxy_set_header X-Forwarded-For     $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Host    $http_host;
+        proxy_set_header X-Forwarded-Server  $http_host;
+        proxy_set_header X-Forwarded-Port    $server_port;
+
+        # Disable buffer to temp files, tweak buffer for memory
+        proxy_max_temp_file_size 0;
+        proxy_buffer_size 64k;
+        proxy_buffers 8 64k;
+        proxy_busy_buffers_size 64k;
+
+        proxy_pass http://sync;
+    }
+
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+}

src/Dockerfile.ui.prod

@@ -1,7 +1,9 @@
 FROM docker.io/node:20 as base
 ARG NODE_ENV
 ARG VITE_API_URL
+ARG VITE_SYNC_URL
 ENV VITE_API_URL=${VITE_API_URL} \
+    VITE_SYNC_URL=${VITE_SYNC_URL} \
     NODE_ENV=${NODE_ENV} \
     PNPM_HOME="/pnpm" \
     PATH="$PNPM_HOME:$PATH"
@@ -27,11 +29,13 @@ FROM docker.io/rclone/rclone:1 as prod
 ARG APP_VERSION
 ARG COMMIT_REF
 ARG VITE_API_URL
+ARG VITE_SYNC_URL
 LABEL org.hotosm.fmtm.app-name="frontend" \
       org.hotosm.fmtm.app-version="${APP_VERSION}" \
       org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \
       org.hotosm.fmtm.maintainer="[email protected]" \
-      org.hotosm.fmtm.api-url="${VITE_API_URL}"
+      org.hotosm.fmtm.api-url="${VITE_API_URL}" \
+      org.hotosm.fmtm.sync-url="${VITE_SYNC_URL}"
 VOLUME /frontend
 COPY container-entrypoint.sh /
 RUN chmod +x /container-entrypoint.sh

src/mapper/src/store/entities.svelte.ts

@@ -11,7 +11,7 @@ function getEntityStatusStream(projectId: number): ShapeStream | undefined {
 		return;
 	}
 	return new ShapeStream({
-		url: 'http://localhost:7055/v1/shape/odk_entities',
+		url: `${import.meta.env.VITE_SYNC_URL}/v1/shape/odk_entities`,
 		where: `project_id=${projectId}`,
 	});
 }

src/mapper/src/store/tasks.svelte.ts

@@ -18,7 +18,7 @@ function getTaskEventStream(projectId: number): ShapeStream | undefined {
 		return;
 	}
 	return new ShapeStream({
-		url: 'http://localhost:7055/v1/shape/task_events',
+		url: `${import.meta.env.VITE_SYNC_URL}/v1/shape/task_events`,
 		where: `project_id=${projectId}`,
 	});
 }