Merge branch 'development' of github.com:hotosm/fmtm into fix/visuali…

…ze-feature

src/backend/app/auth/osm.py

@@ -41,8 +41,8 @@ async def init_osm_auth():
     return Auth(
         osm_url=settings.OSM_URL,
         client_id=settings.OSM_CLIENT_ID,
-        client_secret=settings.OSM_CLIENT_SECRET,
-        secret_key=settings.OSM_SECRET_KEY,
+        client_secret=settings.OSM_CLIENT_SECRET.get_secret_value(),
+        secret_key=settings.OSM_SECRET_KEY.get_secret_value(),
         login_redirect_uri=settings.OSM_LOGIN_REDIRECT_URI,
         scope=settings.OSM_SCOPE,
     )
@@ -108,7 +108,7 @@ def create_jwt_tokens(input_data: dict) -> tuple[str, str]:
     access_token_data = input_data
     access_token = jwt.encode(
         access_token_data,
-        settings.ENCRYPTION_KEY,
+        settings.ENCRYPTION_KEY.get_secret_value(),
         algorithm=settings.JWT_ENCRYPTION_ALGORITHM,
     )
     refresh_token_data = input_data
@@ -117,7 +117,7 @@ def create_jwt_tokens(input_data: dict) -> tuple[str, str]:
     )  # set refresh token expiry to 7 days
     refresh_token = jwt.encode(
         refresh_token_data,
-        settings.ENCRYPTION_KEY,
+        settings.ENCRYPTION_KEY.get_secret_value(),
         algorithm=settings.JWT_ENCRYPTION_ALGORITHM,
     )
 
@@ -130,7 +130,7 @@ def refresh_access_token(payload: dict) -> str:
 
     return jwt.encode(
         payload,
-        settings.ENCRYPTION_KEY,
+        settings.ENCRYPTION_KEY.get_secret_value(),
         algorithm=settings.JWT_ENCRYPTION_ALGORITHM,
     )
 
@@ -150,7 +150,7 @@ def verify_token(token: str):
     try:
         return jwt.decode(
             token,
-            settings.ENCRYPTION_KEY,
+            settings.ENCRYPTION_KEY.get_secret_value(),
             algorithms=[settings.JWT_ENCRYPTION_ALGORITHM],
             audience=settings.FMTM_DOMAIN,
         )

src/backend/app/central/central_crud.py

@@ -53,7 +53,11 @@ def get_odk_project(odk_central: Optional[central_schemas.ODKCentralDecrypted] =
         log.debug("Attempting extraction from environment variables")
         url = settings.ODK_CENTRAL_URL
         user = settings.ODK_CENTRAL_USER
-        pw = settings.ODK_CENTRAL_PASSWD
+        pw = (
+            settings.ODK_CENTRAL_PASSWD.get_secret_value()
+            if settings.ODK_CENTRAL_PASSWD
+            else ""
+        )
 
     try:
         log.debug(f"Connecting to ODKCentral: url={url} user={user}")
@@ -108,7 +112,11 @@ def get_odk_app_user(odk_central: Optional[central_schemas.ODKCentralDecrypted]
         log.debug("Attempting extraction from environment variables")
         url = settings.ODK_CENTRAL_URL
         user = settings.ODK_CENTRAL_USER
-        pw = settings.ODK_CENTRAL_PASSWD
+        pw = (
+            settings.ODK_CENTRAL_PASSWD.get_secret_value()
+            if settings.ODK_CENTRAL_PASSWD
+            else ""
+        )
 
     try:
         log.debug(f"Connecting to ODKCentral: url={url} user={user}")

src/backend/app/config.py

@@ -27,6 +27,7 @@
 from pydantic import (
     BeforeValidator,
     Field,
+    SecretStr,
     TypeAdapter,
     ValidationInfo,
     computed_field,
@@ -119,7 +120,7 @@ class OpenObserveSettings(OtelSettings):
     """Optional OpenTelemetry specific settings (monitoring)."""
 
     OTEL_ENDPOINT: HttpUrlStr = Field(exclude=True)
-    OTEL_AUTH_TOKEN: Optional[str] = Field(exclude=True)
+    OTEL_AUTH_TOKEN: Optional[SecretStr] = Field(exclude=True)
 
     @computed_field
     @property
@@ -135,7 +136,7 @@ def otel_exporter_otlp_headers(self) -> Optional[str]:
         if not self.OTEL_AUTH_TOKEN:
             return None
         # NOTE auth token must be URL encoded, i.e. space=%20
-        auth_header = f"Authorization=Basic%20{self.OTEL_AUTH_TOKEN}"
+        auth_header = f"Authorization=Basic%20{self.OTEL_AUTH_TOKEN.get_secret_value()}"
         os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = auth_header
         return auth_header
 
@@ -150,7 +151,7 @@ class Settings(BaseSettings):
     APP_NAME: str = "FMTM"
     DEBUG: bool = False
     LOG_LEVEL: str = "INFO"
-    ENCRYPTION_KEY: str
+    ENCRYPTION_KEY: SecretStr
     # NOTE HS384 is used for simplicity of implementation and compatibility with
     # existing Fernet based database value encryption
     JWT_ENCRYPTION_ALGORITHM: str = "HS384"
@@ -173,7 +174,7 @@ def assemble_cors_origins(
 
         # Handle localhost/testing scenario
         domain = info.data.get("FMTM_DOMAIN", "fmtm.localhost")
-        dev_port = info.data.get("FMTM_DEV_PORT", "")
+        dev_port = info.data.get("FMTM_DEV_PORT")
         # NOTE fmtm.dev.test is used as the Playwright test domain
         if "localhost" in domain or "fmtm.dev.test" in domain:
             local_server_port = (
@@ -203,7 +204,7 @@ def assemble_cors_origins(
 
     FMTM_DB_HOST: Optional[str] = "fmtm-db"
     FMTM_DB_USER: Optional[str] = "fmtm"
-    FMTM_DB_PASSWORD: Optional[str] = "fmtm"
+    FMTM_DB_PASSWORD: Optional[SecretStr] = "fmtm"
     FMTM_DB_NAME: Optional[str] = "fmtm"
 
     FMTM_DB_URL: Optional[PostgresDsn] = None
@@ -217,19 +218,19 @@ def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any:
         pg_url = PostgresDsn.build(
             scheme="postgresql",
             username=info.data.get("FMTM_DB_USER"),
-            password=info.data.get("FMTM_DB_PASSWORD"),
+            password=info.data.get("FMTM_DB_PASSWORD").get_secret_value(),
             host=info.data.get("FMTM_DB_HOST"),
             path=info.data.get("FMTM_DB_NAME", ""),
         )
         return pg_url
 
     ODK_CENTRAL_URL: Optional[HttpUrlStr] = ""
     ODK_CENTRAL_USER: Optional[str] = ""
-    ODK_CENTRAL_PASSWD: Optional[str] = ""
+    ODK_CENTRAL_PASSWD: Optional[SecretStr] = ""
 
     OSM_CLIENT_ID: str
-    OSM_CLIENT_SECRET: str
-    OSM_SECRET_KEY: str
+    OSM_CLIENT_SECRET: SecretStr
+    OSM_SECRET_KEY: SecretStr
     # NOTE www is required for now
     # https://github.com/openstreetmap/operations/issues/951#issuecomment-1748717154
     OSM_URL: HttpUrlStr = "https://www.openstreetmap.org"
@@ -238,7 +239,7 @@ def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any:
 
     S3_ENDPOINT: str = "http://s3:9000"
     S3_ACCESS_KEY: Optional[str] = ""
-    S3_SECRET_KEY: Optional[str] = ""
+    S3_SECRET_KEY: Optional[SecretStr] = ""
     S3_BUCKET_NAME: str = "fmtm-data"
     S3_DOWNLOAD_ROOT: Optional[str] = None
 
@@ -267,13 +268,14 @@ def configure_s3_download_root(cls, v: Optional[str], info: ValidationInfo) -> s
         else:
             fmtm_domain = info.data.get("FMTM_DOMAIN")
             # Local dev
+            # NOTE for automated tests, this is overridden manually
             if info.data.get("DEBUG"):
                 dev_port = info.data.get("FMTM_DEV_PORT")
                 return f"http://s3.{fmtm_domain}:{dev_port}"
             return f"https://s3.{fmtm_domain}"
 
     RAW_DATA_API_URL: HttpUrlStr = "https://api-prod.raw-data.hotosm.org/v1"
-    RAW_DATA_API_AUTH_TOKEN: Optional[str] = None
+    RAW_DATA_API_AUTH_TOKEN: Optional[SecretStr] = None
 
     @field_validator("RAW_DATA_API_AUTH_TOKEN", mode="before")
     @classmethod
@@ -323,7 +325,7 @@ def get_cipher_suite():
     # we are stuck at 32 char to maintain support with Fernet (reuse the same key).
     #
     # However this would require a migration for all existing instances of FMTM.
-    return Fernet(settings.ENCRYPTION_KEY)
+    return Fernet(settings.ENCRYPTION_KEY.get_secret_value())
 
 
 def encrypt_value(password: Union[str, HttpUrlStr]) -> str:

src/backend/app/db/models.py

@@ -1523,6 +1523,23 @@ async def update(
 
         return updated_task
 
+    @classmethod
+    async def delete(cls, db: Connection, background_task_id: UUID) -> bool:
+        """Delete a background task entry."""
+        sql = """
+            DELETE from background_tasks
+            WHERE id = %(background_task_id)s
+            RETURNING id;
+        """
+
+        async with db.cursor() as cur:
+            await cur.execute(sql, {"background_task_id": background_task_id})
+            success = await cur.fetchone()
+
+        if success:
+            return True
+        return False
+
 
 class DbBasemap(BaseModel):
     """Table tiles_path.
@@ -1636,7 +1653,7 @@ async def create(
     async def update(
         cls,
         db: Connection,
-        basemap_id: int,
+        basemap_id: UUID,
         basemap_update: "BasemapUpdate",
     ) -> Self:
         """Update values for a basemap."""
@@ -1662,6 +1679,23 @@ async def update(
 
         return updated_basemap
 
+    @classmethod
+    async def delete(cls, db: Connection, basemap_id: UUID) -> bool:
+        """Delete a basemap."""
+        sql = """
+            DELETE from basemaps
+            WHERE id = %(basemap_id)s
+            RETURNING id;
+        """
+
+        async with db.cursor() as cur:
+            await cur.execute(sql, {"basemap_id": basemap_id})
+            success = await cur.fetchone()
+
+        if success:
+            return True
+        return False
+
 
 class DbSubmissionPhoto(BaseModel):
     """Table submission_photo.

src/backend/app/db/postgis_utils.py

@@ -800,7 +800,7 @@ def get_osm_geometries(form_category, geometry):
     pg = PostgresClient(
         "underpass",
         extract_config,
-        auth_token=settings.RAW_DATA_API_AUTH_TOKEN
+        auth_token=settings.RAW_DATA_API_AUTH_TOKEN.get_secret_value()
         if settings.RAW_DATA_API_AUTH_TOKEN
         else None,
     )

src/backend/app/organisations/organisation_crud.py

@@ -60,7 +60,9 @@ async def init_admin_org(db: Connection) -> None:
         url="https://hotosm.org",
         odk_central_url=settings.ODK_CENTRAL_URL,
         odk_central_user=settings.ODK_CENTRAL_USER,
-        odk_central_password=settings.ODK_CENTRAL_PASSWD,
+        odk_central_password=settings.ODK_CENTRAL_PASSWD.get_secret_value()
+        if settings.ODK_CENTRAL_PASSWD
+        else "",
         approved=True,
     )
     with open("/opt/app/images/hot-org-logo.png", "rb") as logo_file:

src/backend/app/projects/project_crud.py

@@ -49,7 +49,7 @@
     split_geojson_by_task_areas,
 )
 from app.projects import project_deps, project_schemas
-from app.s3 import add_obj_to_bucket
+from app.s3 import add_file_to_bucket, add_obj_to_bucket
 
 TILESDIR = "/opt/tiles"
 
@@ -151,7 +151,7 @@ async def generate_data_extract(
     pg = PostgresClient(
         "underpass",
         extract_config,
-        auth_token=settings.RAW_DATA_API_AUTH_TOKEN
+        auth_token=settings.RAW_DATA_API_AUTH_TOKEN.get_secret_value()
         if settings.RAW_DATA_API_AUTH_TOKEN
         else None,
     )
@@ -686,16 +686,20 @@ async def get_project_features_geojson(
 def generate_project_basemap(
     db: Connection,
     project_id: int,
+    org_id: int,
     background_task_id: uuid.UUID,
     source: str,
     output_format: str = "mbtiles",
     tms: Optional[str] = None,
 ):
     """Get the tiles for a project.
 
+    FIXME waiting on hotosm/basemap-api project to replace this
+
     Args:
         db (Connection): The database connection.
         project_id (int): ID of project to create tiles for.
+        org_id (int): Organisation ID that the project falls within.
         background_task_id (uuid.UUID): UUID of background task to track.
         source (str): Tile source ("esri", "bing", "google", "custom" (tms)).
         output_format (str, optional): Default "mbtiles".
@@ -711,8 +715,6 @@ def generate_project_basemap(
     # NOTE mbtile max supported zoom level is 22 (in GDAL at least)
     zooms = "12-22" if tms else "12-19"
     tiles_dir = f"{TILESDIR}/{project_id}"
-    # FIXME for now this is still a location on disk and we do not upload to S3
-    # FIXME waiting on basemap-api project to replace this with URL
     outfile = f"{tiles_dir}/{project_id}_{source}tiles.{output_format}"
 
     # NOTE here we put the connection in autocommit mode to ensure we get
@@ -730,7 +732,6 @@ def generate_project_basemap(
                 background_task_id=background_task_id,
                 status=BackgroundTaskStatus.PENDING,
                 tile_source=source,
-                url=outfile,
             ),
         )
 
@@ -761,11 +762,37 @@ def generate_project_basemap(
         )
         log.info(f"Basemap created for project ID {project_id}: {outfile}")
 
+        # Generate S3 urls
+        # We parse as BasemapOut to calculated computed fields (format, mimetype)
+        basemap_out = project_schemas.BasemapOut(
+            **new_basemap.model_dump(exclude=["url"]),
+            url=outfile,
+        )
+        basemap_s3_path = (
+            f"{org_id}/{project_id}/basemaps/{basemap_out.id}.{basemap_out.format}"
+        )
+        log.debug(f"Uploading basemap to S3 path: {basemap_s3_path}")
+        add_file_to_bucket(
+            settings.S3_BUCKET_NAME,
+            basemap_s3_path,
+            outfile,
+            content_type=basemap_out.mimetype,
+        )
+        basemap_external_s3_url = (
+            f"{settings.S3_DOWNLOAD_ROOT}/{settings.S3_BUCKET_NAME}/{basemap_s3_path}"
+        )
+        log.info(f"Upload of basemap to S3 complete: {basemap_external_s3_url}")
+        # Delete file on disk
+        Path(outfile).unlink(missing_ok=True)
+
         update_basemap_sync = async_to_sync(DbBasemap.update)
         update_basemap_sync(
             db,
-            new_basemap.id,
-            project_schemas.BasemapUpdate(status=BackgroundTaskStatus.SUCCESS),
+            basemap_out.id,
+            project_schemas.BasemapUpdate(
+                url=basemap_external_s3_url,
+                status=BackgroundTaskStatus.SUCCESS,
+            ),
         )
 
         update_bg_task_sync = async_to_sync(DbBackgroundTask.update)