Implement ODK Entities for project creation

src/backend/app/db/db_models.py

@@ -263,8 +263,8 @@ class DbProjectChat(Base):
     posted_by = relationship(DbUser, foreign_keys=[user_id])
 
 
-class DbXForm(Base):
-    """Xform templates and custom uploads."""
+class DbXLSForm(Base):
+    """XLSForm templates and custom uploads."""
 
     __tablename__ = "xlsforms"
     id = cast(int, Column(Integer, primary_key=True, autoincrement=True))
@@ -277,6 +277,23 @@ class DbXForm(Base):
     xls = cast(bytes, Column(LargeBinary))  # Human readable representation
 
 
+class DbXForm(Base):
+    """XForms linked per project.
+
+    TODO eventually we will support multiple forms per project.
+    TODO So the category field a stub until then.
+    TODO currently it's maintained under projects.xform_category.
+    """
+
+    __tablename__ = "xforms"
+    id = cast(int, Column(Integer, primary_key=True, autoincrement=True))
+    project_id = cast(
+        int, Column(Integer, ForeignKey("projects.id"), name="project_id", index=True)
+    )
+    odk_form_id = cast(str, Column(String))
+    category = cast(str, Column(String))
+
+
 class DbTaskInvalidationHistory(Base):
     """Information on task invalidation.
 
@@ -449,7 +466,6 @@ class DbTask(Base):
             BigInteger, ForeignKey("users.id", name="fk_users_validator"), index=True
         ),
     )
-    odk_token = cast(str, Column(String, nullable=True))
 
     # Define relationships
     task_history = relationship(
@@ -584,6 +600,10 @@ def tasks_bad(self):
 
     # XForm category specified
     xform_category = cast(str, Column(String))
+    # Linked XForms
+    forms = relationship(
+        DbXForm, backref="project_xform_link", cascade="all, delete, delete-orphan"
+    )
 
     __table_args__ = (
         Index("idx_geometry", outline, postgresql_using="gist"),
@@ -619,6 +639,7 @@ def tasks_bad(self):
     odk_central_url = cast(str, Column(String))
     odk_central_user = cast(str, Column(String))
     odk_central_password = cast(str, Column(String))
+    odk_token = cast(str, Column(String, nullable=True))
 
     form_xls = cast(
         bytes, Column(LargeBinary)

src/backend/app/db/postgis_utils.py

@@ -19,6 +19,7 @@
 
 import json
 import logging
+from asyncio import gather
 from datetime import datetime, timezone
 from random import getrandbits
 from typing import Optional, Union
@@ -28,7 +29,6 @@
 from fastapi import HTTPException
 from geoalchemy2 import WKBElement
 from geoalchemy2.shape import from_shape, to_shape
-from geojson.feature import FeatureCollection
 from geojson_pydantic import Feature, Polygon
 from geojson_pydantic import FeatureCollection as FeatCol
 from shapely.geometry import mapping, shape
@@ -287,18 +287,10 @@ async def split_geojson_by_task_areas(
             ST_SetSRID(ST_GeomFromGeoJSON(feature->>'geometry'), 4326) AS geometry,
             jsonb_set(
                 jsonb_set(
-                    jsonb_set(
-                        feature->'properties',
-                        '{task_id}', to_jsonb(tasks.id), true
-                    ),
-                    '{project_id}', to_jsonb(tasks.project_id), true
+                    feature->'properties',
+                    '{task_id}', to_jsonb(tasks.id), true
                 ),
-                '{title}', to_jsonb(CONCAT(
-                    'project_',
-                    :project_id,
-                    '_task_',
-                    tasks.id
-                )), true
+                '{project_id}', to_jsonb(tasks.project_id), true
             ) AS properties
         FROM (
             SELECT jsonb_array_elements(CAST(:geojson_featcol AS jsonb)->'features')
@@ -357,6 +349,7 @@ async def split_geojson_by_task_areas(
         return None
 
     if feature_collections:
+        # NOTE the feature collections are nested in a tuple, first remove
         task_geojson_dict = {
             record[0]: geojson.loads(json.dumps(record[1]))
             for record in feature_collections
@@ -382,9 +375,9 @@ def add_required_geojson_properties(
             properties["osm_id"] = feature_id
 
         # Check for id type embedded in properties
-        if properties.get("osm_id"):
-            # osm_id exists already, skip
-            pass
+        if osm_id := properties.get("osm_id"):
+            # osm_id property exists, set top level id
+            feature["id"] = osm_id
         else:
             if prop_id := properties.get("id"):
                 # id is nested in properties, use that
@@ -549,8 +542,9 @@ def get_address_from_lat_lon(latitude, longitude):
 
     country = address.get("country", "")
     city = address.get("city", "")
+    state = address.get("state", "")
 
-    address_str = f"{city},{country}"
+    address_str = f"{city},{country}" if city else f"{state},{country}"
 
     if not address_str or address_str == ",":
         log.error("Getting address string failed")
@@ -565,7 +559,7 @@ async def get_address_from_lat_lon_async(latitude, longitude):
 
 
 async def geojson_to_javarosa_geom(geojson_geometry: dict) -> str:
-    """Convert a GeoJSON Polygon geometry to JavaRosa format string.
+    """Convert a GeoJSON geometry to JavaRosa format string.
 
     This format is unique to ODK and the JavaRosa XForm processing library.
     Example JavaRosa polygon (semicolon separated):
@@ -576,59 +570,99 @@ async def geojson_to_javarosa_geom(geojson_geometry: dict) -> str:
     -8.38071535576881 115.640801902838 0.0 0.0
 
     Args:
-        geojson_geometry (dict): The geojson polygon geom.
+        geojson_geometry (dict): The GeoJSON geometry.
 
     Returns:
         str: A string representing the geometry in JavaRosa format.
     """
-    # Ensure the GeoJSON geometry is of type Polygon
-    # FIXME support other geom types
-    if geojson_geometry["type"] != "Polygon":
-        raise ValueError("Input must be a GeoJSON Polygon")
-
-    coordinates = geojson_geometry["coordinates"][
-        0
-    ]  # Extract the coordinates of the Polygon
+    if geojson_geometry is None:
+        return ""
+
+    coordinates = []
+    if geojson_geometry["type"] in ["Point", "LineString", "MultiPoint"]:
+        coordinates = [geojson_geometry.get("coordinates", [])]
+    elif geojson_geometry["type"] in ["Polygon", "MultiLineString"]:
+        coordinates = geojson_geometry.get("coordinates", [])
+    elif geojson_geometry["type"] == "MultiPolygon":
+        # Flatten the list structure to get coordinates of all polygons
+        coordinates = sum(geojson_geometry.get("coordinates", []), [])
+    else:
+        raise ValueError("Unsupported GeoJSON geometry type")
+
     javarosa_geometry = []
+    for polygon in coordinates:
+        for lon, lat in polygon:
+            javarosa_geometry.append(f"{lat} {lon} 0.0 0.0")
 
-    for coordinate in coordinates:
-        lon, lat = coordinate[:2]
-        javarosa_geometry.append(f"{lat} {lon} 0.0 0.0")
+    return ";".join(javarosa_geometry)
 
-    javarosa_geometry_string = ";".join(javarosa_geometry)
 
-    return javarosa_geometry_string
+async def javarosa_to_geojson_geom(javarosa_geom_string: str, geom_type: str) -> dict:
+    """Convert a JavaRosa format string to GeoJSON geometry.
 
+    Args:
+        javarosa_geom_string (str): The JavaRosa geometry.
+        geom_type (str): The geometry type.
 
-async def get_entity_dicts_from_task_geojson(
-    project_id: int,
-    task_id: int,
-    task_data_extract: FeatureCollection,
+    Returns:
+        dict: A geojson geometry.
+    """
+    if javarosa_geom_string is None:
+        return {}
+
+    if geom_type == "Point":
+        lat, lon, _, _ = map(float, javarosa_geom_string.split())
+        geojson_geometry = {"type": "Point", "coordinates": [lon, lat]}
+    elif geom_type == "Polyline":
+        coordinates = [
+            [float(coord) for coord in reversed(point.split()[:2])]
+            for point in javarosa_geom_string.split(";")
+        ]
+        geojson_geometry = {"type": "LineString", "coordinates": coordinates}
+    elif geom_type == "Polygon":
+        coordinates = [
+            [
+                [float(coord) for coord in reversed(point.split()[:2])]
+                for point in coordinate.split(";")
+            ]
+            for coordinate in javarosa_geom_string.split(",")
+        ]
+        geojson_geometry = {"type": "Polygon", "coordinates": coordinates}
+    else:
+        raise ValueError("Unsupported GeoJSON geometry type")
+
+    return geojson_geometry
+
+
+async def feature_geojson_to_entity_dict(
+    feature: dict,
 ) -> dict:
-    """Get a dictionary of Entity info mapped from task geojsons."""
-    id_properties_dict = {}
+    """Convert a single GeoJSON to an Entity dict for upload."""
+    feature_id = feature.get("id")
 
-    features = task_data_extract.get("features", [])
-    for feature in features:
-        geometry = feature.get("geometry")
-        javarosa_geom = await geojson_to_javarosa_geom(geometry)
+    geometry = feature.get("geometry", {})
+    javarosa_geom = await geojson_to_javarosa_geom(geometry)
 
-        properties = feature.get("properties", {})
-        osm_id = properties.get("osm_id", getrandbits(30))
-        tags = properties.get("tags")
-        version = properties.get("version")
-        changeset = properties.get("changeset")
-        timestamp = properties.get("timestamp")
-
-        # Must be string values to work with Entities
-        id_properties_dict[osm_id] = {
-            "project_id": str(project_id),
-            "task_id": str(task_id),
-            "geometry": javarosa_geom,
-            "tags": str(tags),
-            "version": str(version),
-            "changeset": str(changeset),
-            "timestamp": str(timestamp),
-        }
+    # NOTE all properties MUST be string values for Entities, convert
+    properties = {
+        str(key): str(value) for key, value in feature.get("properties", {}).items()
+    }
+
+    task_id = properties.get("task_id")
+    entity_label = f"task {task_id} feature {feature_id}"
+
+    return {entity_label: {"geometry": javarosa_geom, **properties}}
+
+
+async def task_geojson_dict_to_entity_values(task_geojson_dict):
+    """Convert a dict of task GeoJSONs into data for ODK Entity upload."""
+    asyncio_tasks = []
+    for _, geojson_dict in task_geojson_dict.items():
+        features = geojson_dict.get("features", [])
+        asyncio_tasks.extend(
+            [feature_geojson_to_entity_dict(feature) for feature in features]
+        )
 
-    return id_properties_dict
+    entity_values = await gather(*asyncio_tasks)
+    # Merge all dicts into a single dict
+    return {k: v for result in entity_values for k, v in result.items()}

src/backend/app/helpers/helper_routes.py

@@ -18,6 +18,8 @@
 """Routes to help with common processes in the FMTM workflow."""
 
 import json
+from io import BytesIO
+from pathlib import Path
 
 from fastapi import (
     APIRouter,
@@ -28,11 +30,17 @@
 from fastapi.responses import Response
 
 from app.auth.osm import AuthUser, login_required
+from app.central.central_crud import (
+    convert_geojson_to_odk_csv,
+    convert_odk_submission_json_to_geojson,
+    read_and_test_xform,
+)
 from app.db.postgis_utils import (
     add_required_geojson_properties,
+    javarosa_to_geojson_geom,
     parse_and_filter_geojson,
 )
-from app.models.enums import HTTPStatus
+from app.models.enums import GeometryType, HTTPStatus
 
 router = APIRouter(
     prefix="/helper",
@@ -72,3 +80,83 @@ async def append_required_geojson_properties(
         status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
         detail="Your geojson file is invalid.",
     )
+
+
+@router.post("/convert-xlsform-to-xform")
+async def convert_xlsform_to_xform(
+    xlsform: UploadFile,
+    current_user: AuthUser = Depends(login_required),
+):
+    """Convert XLSForm to XForm XML."""
+    filename = Path(xlsform.filename)
+    file_ext = filename.suffix.lower()
+
+    allowed_extensions = [".xls", ".xlsx"]
+    if file_ext not in allowed_extensions:
+        raise HTTPException(
+            status_code=400, detail="Provide a valid .xls or .xlsx file"
+        )
+
+    contents = await xlsform.read()
+    xform_data = await read_and_test_xform(
+        BytesIO(contents), file_ext, return_form_data=True
+    )
+
+    headers = {"Content-Disposition": f"attachment; filename={filename.stem}.xml"}
+    return Response(xform_data.getvalue(), headers=headers)
+
+
+@router.post("/convert-geojson-to-odk-csv")
+async def convert_geojson_to_odk_csv_wrapper(
+    geojson: UploadFile,
+    current_user: AuthUser = Depends(login_required),
+):
+    """Convert GeoJSON upload media to ODK CSV upload media."""
+    filename = Path(geojson.filename)
+    file_ext = filename.suffix.lower()
+
+    allowed_extensions = [".json", ".geojson"]
+    if file_ext not in allowed_extensions:
+        raise HTTPException(
+            status_code=400, detail="Provide a valid .json or .geojson file"
+        )
+
+    contents = await geojson.read()
+    feature_csv = await convert_geojson_to_odk_csv(BytesIO(contents))
+
+    headers = {"Content-Disposition": f"attachment; filename={filename.stem}.csv"}
+    return Response(feature_csv.getvalue(), headers=headers)
+
+
+@router.post("/javarosa-geom-to-geojson")
+async def convert_javarosa_geom_to_geojson(
+    javarosa_string: str,
+    geometry_type: GeometryType,
+    current_user: AuthUser = Depends(login_required),
+):
+    """Convert a JavaRosa geometry string to GeoJSON."""
+    return await javarosa_to_geojson_geom(javarosa_string, geometry_type)
+
+
+@router.post("/convert-odk-submission-json-to-geojson")
+async def convert_odk_submission_json_to_geojson_wrapper(
+    json_file: UploadFile,
+    current_user: AuthUser = Depends(login_required),
+):
+    """Convert the ODK submission output JSON to GeoJSON.
+
+    The submission JSON be downloaded via ODK Central, or osm-fieldwork.
+    The logic works with the standardised XForm form fields from osm-fieldwork.
+    """
+    filename = Path(json_file.filename)
+    file_ext = filename.suffix.lower()
+
+    allowed_extensions = [".json"]
+    if file_ext not in allowed_extensions:
+        raise HTTPException(status_code=400, detail="Provide a valid .json file")
+
+    contents = await json_file.read()
+    submission_geojson = await convert_odk_submission_json_to_geojson(BytesIO(contents))
+
+    headers = {"Content-Disposition": f"attachment; filename={filename.stem}.geojson"}
+    return Response(submission_geojson.getvalue(), headers=headers)