Add OpenTelemetry configuration for backend API (monitoring)

.env.example

@@ -44,3 +44,14 @@ FMTM_DB_NAME=${FMTM_DB_NAME:-"fmtm"}
 
 ### Underpass (optional override) ###
 UNDERPASS_API_URL=${UNDERPASS_API_URL:-"https://api-prod.raw-data.hotosm.org/v1"}
+
+# Monitoring (OpenTelemetry). Options: 'openobserve', 'sentry'.
+MONITORING=${MONITORING}
+# Monitoring (OpenObserve)
+OPENOBSERVE_USER=${OTEL_USERNAME:-${ODK_CENTRAL_USER:-"[email protected]"}}
+OPENOBSERVE_PASSWORD=${OPENOBSERVE_PASSWORD:-"testuserpassword"}
+OPENOBSERVE_RETENTION_DAYS=${OPENOBSERVE_RETENTION_DAYS:-90}
+OTEL_ENDPOINT=${OTEL_ENDPOINT}
+OTEL_AUTH_TOKEN=${OTEL_AUTH_TOKEN}
+# Monitoring (Sentry)
+SENTRY_DSN=${SENTRY_DSN}

.github/workflows/build_and_deploy.yml

@@ -17,7 +17,7 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend
@@ -29,20 +29,20 @@ jobs:
     secrets: inherit
 
   frontend-tests:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       working_dir: src/frontend
 
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     needs: [pytest]
     with:
       context: src/backend
       build_target: prod
       image_name: ghcr.io/${{ github.repository }}/backend
 
   frontend-build:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     needs: [frontend-tests]
     with:
       context: src/frontend
@@ -152,7 +152,7 @@ jobs:
     needs:
       - smoke-test-backend
       - smoke-test-frontend
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       environment: ${{ github.ref_name }}
       docker_compose_file: "docker-compose.${{ github.ref_name }}.yml"

.github/workflows/build_ci_img.yml

@@ -16,7 +16,7 @@ on:
 
 jobs:
   backend-ci-build:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: src/backend
       build_target: ci

.github/workflows/build_odk_imgs.yml

@@ -13,7 +13,7 @@ on:
 
 jobs:
   build-odkcentral:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: odkcentral/api
       image_tags: |
@@ -26,7 +26,7 @@ jobs:
       # multi_arch: true
 
   build-odkcentral-ui:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: odkcentral/ui
       image_tags: |

.github/workflows/build_proxy_imgs.yml

@@ -10,7 +10,7 @@ on:
 
 jobs:
   build-cert-init-main:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: nginx
       build_target: certs-init-main
@@ -21,7 +21,7 @@ jobs:
       multi_arch: true
 
   build-cert-init-dev:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: nginx
       build_target: certs-init-development
@@ -33,7 +33,7 @@ jobs:
       multi_arch: true
 
   build-proxy-main:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: nginx
       build_target: main
@@ -44,7 +44,7 @@ jobs:
       multi_arch: true
 
   build-proxy-dev:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: nginx
       build_target: development

.github/workflows/docs.yml

@@ -12,19 +12,19 @@ on:
 
 jobs:
   build_doxygen:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       output_path: docs/apidocs
 
   build_openapi_json:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}
       example_env_file_path: ".env.example"
       output_path: docs/openapi.json
 
   publish_docs:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     needs:
       - build_doxygen
       - build_openapi_json

.github/workflows/pr_test_backend.yml

@@ -14,7 +14,7 @@ on:
 
 jobs:
   pytest:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       image_name: ghcr.io/${{ github.repository }}/backend
       build_context: src/backend

.github/workflows/pr_test_frontend.yml

@@ -14,7 +14,7 @@ on:
 
 jobs:
   unit-tests:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       working_dir: src/frontend
 

.github/workflows/tag_build.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   backend-build:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       context: src/backend
       build_target: prod

.github/workflows/wiki.yml

@@ -10,6 +10,6 @@ on:
 
 jobs:
   publish-docs-to-wiki:
-    uses: hotosm/gh-workflows/.github/workflows/[email protected].0
+    uses: hotosm/gh-workflows/.github/workflows/[email protected].1
     with:
       homepage_path: "wiki_redirect.md"

INSTALL.md

@@ -62,7 +62,7 @@ If more details are required, check out the
       - [Build the Frontend](#build-the-frontend)
       - [Start the Containers](#start-the-containers)
     - [Setup ODK Central User (Optional)](#setup-odk-central-user-optional)
-    - [Import Test Data (Optional)](#import-test-data-optional)
+    - [Set Up Monitoring (Optional)](#set-up-monitoring-optional)
     - [Check Authentication (Optional)](#check-authentication-optional)
   - [Alternative Operating Systems](#alternative-operating-systems)
     - [Windows](#windows)
@@ -218,15 +218,21 @@ docker-compose exec central odk-cmd --email [email protected] user-promo
 
 > Note: Alternatively, you may use an external Central server and user.
 
-### Import Test Data (Optional)
+### Set Up Monitoring (Optional)
 
-If running a local test version, test data is available to get started quickly.
+- There is an optional configuration for application monitoring via OpenTelemetry.
+- To enable this set in your `.env` file:
 
-- Navigate to the `import-test-data` endpoint in the API docs page:
+  ```dotenv
+  # For OpenObserve
+  MONITORING="openobserve"
+  # For Sentry
+  MONITORING="sentry"
+  ```
 
-<http://api.fmtm.localhost:7050/docs#/debug/import_test_data_debug_import_test_data_get>
-
-- Click `Try it out`, then `execute`.
+- Check the `.env.example` for additional required parameters.
+- Everything should be configured for you to see endpoint calls in the
+  selected web dashboard, providing full error tracebacks and stats.
 
 ### Check Authentication (Optional)
 

docker-compose.yml

@@ -30,7 +30,7 @@ volumes:
 
 networks:
   fmtm-net:
-    name: fmtm-local
+    name: fmtm-${GIT_BRANCH:-local}
 
 services:
   proxy:

observability/openobserve/docker-compose.yml

@@ -0,0 +1,51 @@
+# Copyright (c) 2022, 2023 Humanitarian OpenStreetMap Team
+# This file is part of FMTM.
+#
+#     FMTM is free software: you can redistribute it and/or modify
+#     it under the terms of the GNU General Public License as published by
+#     the Free Software Foundation, either version 3 of the License, or
+#     (at your option) any later version.
+#
+#     FMTM is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#     GNU General Public License for more details.
+#
+#     You should have received a copy of the GNU General Public License
+#     along with FMTM.  If not, see <https:#www.gnu.org/licenses/>.
+#
+
+version: "3"
+
+volumes:
+  fmtm_monitoring_data:
+    name: fmtm-monitoring-data-${GIT_BRANCH:-local}
+
+networks:
+  fmtm-net:
+    name: fmtm-${GIT_BRANCH:-local}
+
+services:
+  openobserve:
+    image: public.ecr.aws/zinclabs/openobserve:latest
+    environment:
+      ZO_ROOT_USER_EMAIL: ${OPENOBSERVE_USER}
+      ZO_ROOT_USER_PASSWORD: ${OPENOBSERVE_PASSWORD}
+      # Keep data 90 days
+      ZO_COMPACT_DATA_RETENTION_DAYS: ${OPENOBSERVE_RETENTION_DAYS}
+      # S3 config
+      # ZO_LOCAL_MODE_STORAGE: s3
+      # ZO_S3_SERVER_URL: ${S3_xx}
+      # ZO_S3_REGION_NAME: us-west-1
+      # ZO_S3_ACCESS_KEY: ${S3_xx}
+      # ZO_S3_SECRET_KEY: ${S3_xx}
+      # ZO_S3_BUCKET_NAME: ${S3_xx}
+      # ZO_S3_PROVIDER: s3
+    ports:
+      - "5080:5080"
+    volumes:
+      - fmtm_monitoring_data:/data
+    networks:
+      - fmtm-net
+    restart: unless-stopped
+    # TODO add healthcheck

src/backend/Dockerfile

@@ -52,14 +52,20 @@ COPY pyproject.toml pdm.lock /opt/python/
 RUN pip install --no-cache-dir --upgrade pip \
     && pip install --no-cache-dir pdm==2.6.1
 RUN pdm export --prod > requirements.txt \
-    && pdm export -G debug --no-default \
-    --without-hashes > requirements-debug.txt \
-    && pdm export -G test -G docs -G dev --no-default \
-    --without-hashes > requirements-ci.txt
+    # Export with default deps, as we install one or the other
+    && pdm export -G monitoring \
+    --without-hashes > requirements-monitoring.txt \
+    && pdm export -G debug \
+    --no-default --without-hashes > requirements-debug.txt \
+    && pdm export -G test -G docs -G dev \
+    --no-default --without-hashes > requirements-ci.txt
 
 
 # Build stage will all dependencies required to build Python wheels
 FROM base as build
+# NOTE this argument is specified during production build on Github workflow
+# NOTE only the production API image contains the monitoring dependencies
+ARG WITH_MONITORING
 RUN set -ex \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install \
@@ -74,9 +80,16 @@ RUN set -ex \
         "git" \
     && rm -rf /var/lib/apt/lists/*
 COPY --from=extract-deps \
-    /opt/python/requirements.txt /opt/python/
-RUN pip install --user --no-warn-script-location \
-    --no-cache-dir -r /opt/python/requirements.txt
+    /opt/python/requirements.txt \
+    /opt/python/requirements-monitoring.txt \
+    /opt/python/
+# Install with or without monitoring, depending on flag
+RUN pip install --user --no-warn-script-location --no-cache-dir \
+    $(if [ "$WITH_MONITORING" = "true" ]; then \
+        echo "-r /opt/python/requirements-monitoring.txt"; \
+    else \
+        echo "-r /opt/python/requirements.txt"; \
+    fi)
 
 
 # Run stage will minimal dependencies required to run Python libraries