diff --git a/docker-compose.development.yml b/docker-compose.development.yml index 0f25e694ca..19e70a1855 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -59,6 +59,8 @@ services: condition: service_completed_successfully ui: condition: service_completed_successfully + electric: + condition: service_started volumes: - fmtm_frontend:/usr/share/nginx/html/fmtm - central_frontend:/usr/share/nginx/html/central/ @@ -69,6 +71,7 @@ services: FMTM_API_DOMAIN: ${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}} FMTM_ODK_DOMAIN: ${FMTM_ODK_DOMAIN:-odk.${FMTM_DOMAIN}} FMTM_S3_DOMAIN: ${FMTM_S3_DOMAIN:-s3.${FMTM_DOMAIN}} + FMTM_SYNC_DOMAIN: ${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}} ports: - 80:80 - 443:443 @@ -119,6 +122,7 @@ services: args: APP_VERSION: ${GIT_BRANCH} VITE_API_URL: https://${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}} + VITE_SYNC_URL: https://${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}} NODE_ENV: development volumes: - fmtm_frontend:/frontend @@ -216,6 +220,22 @@ services: timeout: 5s retries: 3 + electric: + image: "electricsql/electric:${ELECTRIC_TAG:-0.7.7}" + depends_on: + fmtm-db: + condition: service_healthy + migrations: + condition: service_completed_successfully + environment: + DATABASE_URL: postgresql://${FMTM_DB_USER}:${FMTM_DB_PASSWORD}@${FMTM_DB_HOST:-fmtm-db}/${FMTM_DB_NAME:-fmtm}?sslmode=disable + # OTEL_EXPORT: otlp + # OTLP_ENDPOINT: https://... + # ELECTRIC_WRITE_TO_PG_MODE: direct_writes + networks: + - fmtm-net + restart: "unless-stopped" + central-db: image: "postgis/postgis:${POSTGIS_TAG:-14-3.5-alpine}" volumes: diff --git a/docker-compose.main.yml b/docker-compose.main.yml index 7d082399e5..dcd483b7df 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -51,6 +51,8 @@ services: condition: service_started ui: condition: service_completed_successfully + electric: + condition: service_started volumes: - fmtm_frontend:/usr/share/nginx/html/fmtm - certs:/etc/letsencrypt @@ -59,6 +61,7 @@ services: FMTM_DOMAIN: ${FMTM_DOMAIN} FMTM_API_DOMAIN: ${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}} FMTM_S3_DOMAIN: ${FMTM_S3_DOMAIN:-s3.${FMTM_DOMAIN}} + FMTM_SYNC_DOMAIN: ${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}} ports: - 80:80 - 443:443 @@ -109,6 +112,7 @@ services: args: APP_VERSION: main VITE_API_URL: https://${FMTM_API_DOMAIN:-api.${FMTM_DOMAIN}} + VITE_SYNC_URL: https://${FMTM_SYNC_DOMAIN:-sync.${FMTM_DOMAIN}} NODE_ENV: production volumes: - fmtm_frontend:/frontend @@ -156,6 +160,22 @@ services: timeout: 5s retries: 3 + electric: + image: "electricsql/electric:${ELECTRIC_TAG:-0.7.7}" + depends_on: + fmtm-db: + condition: service_healthy + migrations: + condition: service_completed_successfully + environment: + DATABASE_URL: postgresql://${FMTM_DB_USER}:${FMTM_DB_PASSWORD}@${FMTM_DB_HOST:-fmtm-db}/${FMTM_DB_NAME:-fmtm}?sslmode=disable + # OTEL_EXPORT: otlp + # OTLP_ENDPOINT: https://... + # ELECTRIC_WRITE_TO_PG_MODE: direct_writes + networks: + - fmtm-net + restart: "unless-stopped" + migrations: image: "ghcr.io/hotosm/fmtm/backend:main" depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index 1ab0c5fbbe..2d44175584 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,8 @@ services: required: false s3: condition: service_started + electric: + condition: service_started volumes: - central_frontend:/usr/share/nginx/html/central ports: @@ -153,6 +155,7 @@ services: # - ../ui:/app/node_modules/@hotosm/ui:ro environment: - VITE_API_URL=${API_URL:-http://api.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050}} + - VITE_SYNC_URL=http://sync.${FMTM_DOMAIN}:${FMTM_DEV_PORT:-7050} networks: - fmtm-net restart: "unless-stopped" @@ -280,13 +283,7 @@ services: DATABASE_URL: postgresql://${FMTM_DB_USER:-fmtm}:${FMTM_DB_PASSWORD:-fmtm}@${FMTM_DB_HOST:-fmtm-db}/${FMTM_DB_NAME:-fmtm}?sslmode=disable # OTEL_EXPORT: otlp # OTLP_ENDPOINT: https://... - # ELECTRIC_WRITE_TO_PG_MODE: direct_writes - # AUTH_JWT_ALG: HS384 - # AUTH_JWT_KEY: ${ENCRYPTION_KEY} - # AUTH_JWT_AUD: ${FMTM_DOMAIN} - ports: - - "7055:3000" networks: - fmtm-net restart: "unless-stopped" diff --git a/docs/dev/Production.md b/docs/dev/Production.md index 655c09a0b6..c228b8efaf 100644 --- a/docs/dev/Production.md +++ b/docs/dev/Production.md @@ -77,6 +77,7 @@ These defaults can be overridden with respective environment variables: FMTM_API_DOMAIN FMTM_ODK_DOMAIN FMTM_S3_DOMAIN +FMTM_SYNC_DOMAIN ``` ### Connecting to a remote database diff --git a/nginx/Dockerfile b/nginx/Dockerfile index 3998bb5042..ecd3f55a9a 100644 --- a/nginx/Dockerfile +++ b/nginx/Dockerfile @@ -92,6 +92,7 @@ COPY templates/dev/fmtm.conf.template \ templates/dev/api.conf.template \ templates/dev/odk.conf.template \ templates/dev/minio.conf.template \ + templates/dev/sync.conf.template \ /etc/nginx/templates/ @@ -103,6 +104,7 @@ RUN chmod +x /docker-entrypoint.sh COPY templates/cert-init/fmtm.conf.template \ templates/cert-init/api.conf.template \ templates/cert-init/minio.conf.template \ + templates/cert-init/sync.conf.template \ /etc/nginx/templates/ @@ -118,6 +120,7 @@ FROM base as main COPY templates/fmtm.conf.template \ templates/api.conf.template \ templates/minio.conf.template \ + templates/sync.conf.template \ /etc/nginx/templates/ diff --git a/nginx/container-entrypoint.sh b/nginx/container-entrypoint.sh index b18eb462c1..854c1dd2d7 100644 --- a/nginx/container-entrypoint.sh +++ b/nginx/container-entrypoint.sh @@ -62,6 +62,12 @@ if [ -n "${FMTM_S3_DOMAIN}" ]; then certbot_args+=("-d" "${FMTM_S3_DOMAIN}") fi +# Add FMTM_SYNC_DOMAIN if present +if [ -n "${FMTM_SYNC_DOMAIN}" ]; then + echo "Adding ${FMTM_SYNC_DOMAIN} to certificate for domain ${FMTM_DOMAIN}." + certbot_args+=("-d" "${FMTM_SYNC_DOMAIN}") +fi + # Run certbot with the constructed arguments echo "Running command: certbot --non-interactive certonly ${certbot_args[*]}" certbot --non-interactive certonly "${certbot_args[@]}" diff --git a/nginx/templates/cert-init/sync.conf.template b/nginx/templates/cert-init/sync.conf.template new file mode 100644 index 0000000000..4968d88ecc --- /dev/null +++ b/nginx/templates/cert-init/sync.conf.template @@ -0,0 +1,26 @@ +# Copyright (c) Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +server { + listen 80; + server_name ${FMTM_SYNC_DOMAIN}; + + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } +} diff --git a/nginx/templates/dev/sync.conf.template b/nginx/templates/dev/sync.conf.template new file mode 100644 index 0000000000..4f40e9763a --- /dev/null +++ b/nginx/templates/dev/sync.conf.template @@ -0,0 +1,62 @@ +# Copyright (c) Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream sync { + server electric:3000 max_fails=1 fail_timeout=2s; + keepalive 32; +} + +server { + # Default handler for port 80 + listen 80; + server_name sync.fmtm.localhost; + + # Max upload size 10MB + client_max_body_size 10M; + + location / { + # Max time to initiate connection with electric + proxy_connect_timeout 10s; + # Max time for a backend response to return + proxy_read_timeout 20s; + # Max time to send request to backend, i.e. upload + proxy_send_timeout 20s; + + # Requests headers + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $http_host; + proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + proxy_pass http://sync; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/nginx/templates/sync.conf.template b/nginx/templates/sync.conf.template new file mode 100644 index 0000000000..2a6ced770a --- /dev/null +++ b/nginx/templates/sync.conf.template @@ -0,0 +1,79 @@ +# Copyright (c) Humanitarian OpenStreetMap Team +# +# This file is part of FMTM. +# +# FMTM is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# FMTM is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with FMTM. If not, see . +# + +upstream sync { + server electric:3000 max_fails=1 fail_timeout=2s; + keepalive 32; +} + +server { + # Default handler for port 80 + listen 80; + server_name ${FMTM_SYNC_DOMAIN}; + return 301 https://$host$request_uri; +} + +server { + # Default handler for port 443 + listen 443 ssl reuseport; + server_name ${FMTM_SYNC_DOMAIN}; + + ssl_certificate /etc/letsencrypt/live/${FMTM_DOMAIN}/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/${FMTM_DOMAIN}/privkey.pem; + include /etc/nginx/options-ssl-nginx.conf; + include /etc/nginx/options-security.conf; + + # Max upload size 1GB + client_max_body_size 1G; + + # Response headers (Access-Control-Allow-Origin set by FastAPI, not required) + add_header 'Content-Security-Policy' 'upgrade-insecure-requests'; + # For opentelemetry + add_header 'Access-Control-Allow-Headers' 'traceparent,tracestate'; + + location / { + # Max time to initiate connection with electric + proxy_connect_timeout 10s; + # Max time for a backend response to return + proxy_read_timeout 20s; + # Max time to send request to backend, i.e. upload + proxy_send_timeout 20s; + + # Requests headers + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Host $http_host; + proxy_set_header X-Forwarded-Server $http_host; + proxy_set_header X-Forwarded-Port $server_port; + + # Disable buffer to temp files, tweak buffer for memory + proxy_max_temp_file_size 0; + proxy_buffer_size 64k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 64k; + + proxy_pass http://sync; + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} diff --git a/src/Dockerfile.ui.prod b/src/Dockerfile.ui.prod index 9f9f9ea295..98a60274d7 100644 --- a/src/Dockerfile.ui.prod +++ b/src/Dockerfile.ui.prod @@ -1,7 +1,9 @@ FROM docker.io/node:20 as base ARG NODE_ENV ARG VITE_API_URL +ARG VITE_SYNC_URL ENV VITE_API_URL=${VITE_API_URL} \ + VITE_SYNC_URL=${VITE_SYNC_URL} \ NODE_ENV=${NODE_ENV} \ PNPM_HOME="/pnpm" \ PATH="$PNPM_HOME:$PATH" @@ -27,11 +29,13 @@ FROM docker.io/rclone/rclone:1 as prod ARG APP_VERSION ARG COMMIT_REF ARG VITE_API_URL +ARG VITE_SYNC_URL LABEL org.hotosm.fmtm.app-name="frontend" \ org.hotosm.fmtm.app-version="${APP_VERSION}" \ org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ org.hotosm.fmtm.maintainer="sysadmin@hotosm.org" \ - org.hotosm.fmtm.api-url="${VITE_API_URL}" + org.hotosm.fmtm.api-url="${VITE_API_URL}" \ + org.hotosm.fmtm.sync-url="${VITE_SYNC_URL}" VOLUME /frontend COPY container-entrypoint.sh / RUN chmod +x /container-entrypoint.sh diff --git a/src/mapper/src/store/entities.svelte.ts b/src/mapper/src/store/entities.svelte.ts index 613f6483c2..7767e091e3 100644 --- a/src/mapper/src/store/entities.svelte.ts +++ b/src/mapper/src/store/entities.svelte.ts @@ -11,7 +11,7 @@ function getEntityStatusStream(projectId: number): ShapeStream | undefined { return; } return new ShapeStream({ - url: 'http://localhost:7055/v1/shape/odk_entities', + url: `${import.meta.env.VITE_SYNC_URL}/v1/shape/odk_entities`, where: `project_id=${projectId}`, }); } diff --git a/src/mapper/src/store/tasks.svelte.ts b/src/mapper/src/store/tasks.svelte.ts index 74a8250528..daeb22e35f 100644 --- a/src/mapper/src/store/tasks.svelte.ts +++ b/src/mapper/src/store/tasks.svelte.ts @@ -18,7 +18,7 @@ function getTaskEventStream(projectId: number): ShapeStream | undefined { return; } return new ShapeStream({ - url: 'http://localhost:7055/v1/shape/task_events', + url: `${import.meta.env.VITE_SYNC_URL}/v1/shape/task_events`, where: `project_id=${projectId}`, }); }