feat: add link for interactive xlsform editing during project creation (

#1480)

* fix(backend): update logic to read/insert xlsforms at startup

* fix(backend): check that file exists before returning template form

* fix: add xlsforms.fmtm.dev to extra cors origins

* feat(frontend): update logic for custom checkbox, add disabled prop

* feat(frontend): add link for interactive xlsform editing in proj creation

src/backend/app/config.py

@@ -178,6 +178,8 @@ def assemble_cors_origins(
         if frontend_domain := info.data.get("FMTM_DOMAIN"):
             default_origins = [
                 f"{url_scheme}://{frontend_domain}{local_server_port}",
+                # Also add the xlsform-editor url
+                "https://xlsforms.fmtm.dev",
             ]
 
         if val is None:

src/backend/app/db/db_models.py

@@ -271,10 +271,7 @@ class DbXLSForm(Base):
     # The XLSForm name is the only unique thing we can use for a key
     # so on conflict update works. Otherwise we get multiple entries.
     title = cast(str, Column(String, unique=True))
-    category = cast(str, Column(String))
-    description = cast(str, Column(String))
-    xml = cast(str, Column(String))  # Internal form representation
-    xls = cast(bytes, Column(LargeBinary))  # Human readable representation
+    xls = cast(bytes, Column(LargeBinary))
 
 
 class DbXForm(Base):

src/backend/app/helpers/helper_routes.py

@@ -60,11 +60,10 @@
 @router.get("/download-template-xlsform")
 async def download_template(
     category: XLSFormType,
-    current_user: AuthUser = Depends(login_required),
 ):
     """Download an XLSForm template to fill out."""
     xlsform_path = f"{xlsforms_path}/{category}.xls"
-    if Path(xlsform_path).exists:
+    if Path(xlsform_path).exists():
         return FileResponse(xlsform_path, filename="form.xls")
     else:
         raise HTTPException(status_code=HTTPStatus.NOT_FOUND, detail="Form not found")

src/backend/app/main.py

@@ -48,7 +48,7 @@
 from app.organisations import organisation_routes
 from app.organisations.organisation_crud import init_admin_org
 from app.projects import project_routes
-from app.projects.project_crud import read_xlsforms
+from app.projects.project_crud import read_and_insert_xlsforms
 from app.submissions import submission_routes
 from app.tasks import tasks_routes
 from app.users import user_routes
@@ -89,7 +89,7 @@ async def lifespan(
     log.debug("Initialising admin org and user in DB.")
     await init_admin_org(db_conn)
     log.debug("Reading XLSForms from DB.")
-    await read_xlsforms(db_conn, xlsforms_path)
+    await read_and_insert_xlsforms(db_conn, xlsforms_path)
 
     yield
 

src/backend/app/projects/project_crud.py

@@ -18,12 +18,11 @@
 """Logic for FMTM project routes."""
 
 import json
-import os
 import subprocess
 import uuid
 from asyncio import gather
-from importlib.resources import files as pkg_files
 from io import BytesIO
+from pathlib import Path
 from typing import List, Optional, Union
 
 import geoalchemy2
@@ -43,8 +42,7 @@
 from osm_fieldwork.xlsforms import entities_registration, xlsforms_path
 from osm_rawdata.postgres import PostgresClient
 from shapely.geometry import shape
-from sqlalchemy import and_, column, func, inspect, select, table, text
-from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy import and_, column, func, select, table, text
 from sqlalchemy.orm import Session
 
 from app.central import central_crud, central_deps
@@ -524,48 +522,58 @@ async def split_geojson_into_tasks(
 # ---------------------------
 
 
-async def read_xlsforms(
-    db: Session,
-    directory: str,
-):
-    """Read the list of XLSForms from the disk."""
-    xlsforms = list()
-    package_name = "osm_fieldwork"
-    package_files = pkg_files(package_name)
-    for xls in os.listdir(directory):
-        if xls.endswith(".xls") or xls.endswith(".xlsx"):
-            file_name = xls.split(".")[0]
-            yaml_file_name = f"data_models/{file_name}.yaml"
-            if package_files.joinpath(yaml_file_name).is_file():
-                xlsforms.append(xls)
-            else:
-                continue
-
-    inspect(db_models.DbXLSForm)
-    forms = table(
-        "xlsforms", column("title"), column("xls"), column("xml"), column("id")
+async def read_and_insert_xlsforms(db, directory):
+    """Read the list of XLSForms from the disk and insert to DB."""
+    existing_titles = set(
+        title for (title,) in db.query(db_models.DbXLSForm.title).all()
     )
-    # x = Table('xlsforms', MetaData())
-    # x.primary_key.columns.values()
-
-    for xlsform in xlsforms:
-        infile = f"{directory}/{xlsform}"
-        if os.path.getsize(infile) <= 0:
-            log.warning(f"{infile} is empty!")
+    xlsforms_on_disk = [
+        file.stem
+        for file in Path(directory).glob("*.xls")
+        if not file.stem.startswith("entities")
+    ]
+
+    # Insert new XLSForms to DB and update existing ones
+    for xlsform_name in xlsforms_on_disk:
+        file_path = Path(directory) / f"{xlsform_name}.xls"
+
+        if file_path.stat().st_size == 0:
+            log.warning(f"{file_path} is empty!")
             continue
-        xls = open(infile, "rb")
-        name = xlsform.split(".")[0]
-        data = xls.read()
-        xls.close()
-        # log.info(xlsform)
-        ins = insert(forms).values(title=name, xls=data)
-        sql = ins.on_conflict_do_update(
-            constraint="xlsforms_title_key", set_=dict(title=name, xls=data)
+
+        with open(file_path, "rb") as xls:
+            data = xls.read()
+
+        try:
+            insert_query = text(
+                """
+                INSERT INTO xlsforms (title, xls)
+                VALUES (:title, :xls)
+                ON CONFLICT (title) DO UPDATE SET
+                title = EXCLUDED.title, xls = EXCLUDED.xls
+            """
+            )
+            db.execute(insert_query, {"title": xlsform_name, "xls": data})
+            db.commit()
+            log.info(f"Inserted or updated {xlsform_name} xlsform to database")
+
+        except Exception as e:
+            log.error(
+                f"Failed to insert or update {xlsform_name} in the database. Error: {e}"
+            )
+
+    # Delete XLSForms from DB that are not found on disk
+    for title in existing_titles - set(xlsforms_on_disk):
+        delete_query = text(
+            """
+            DELETE FROM xlsforms WHERE title = :title
+        """
         )
-        db.execute(sql)
+        db.execute(delete_query, {"title": title})
         db.commit()
+        log.info(f"Deleted {title} from the database as it was not found on disk.")
 
-    return xlsforms
+    return xlsforms_on_disk
 
 
 async def get_odk_id_for_project(db: Session, project_id: int):

src/backend/migrations/init/fmtm_base_schema.sql

@@ -540,9 +540,6 @@ ALTER TABLE public.users OWNER TO fmtm;
 CREATE TABLE public.xlsforms (
     id integer NOT NULL,
     title character varying,
-    category character varying,
-    description character varying,
-    xml character varying,
     xls bytea
 );
 ALTER TABLE public.xlsforms OWNER TO fmtm;

src/frontend/src/components/common/Checkbox.tsx

@@ -9,6 +9,7 @@ type CustomCheckboxType = {
   onCheckedChange: (checked: boolean) => void;
   className?: string;
   labelClickable?: boolean;
+  disabled?: boolean;
 };
 
 const Checkbox = React.forwardRef<
@@ -19,6 +20,7 @@ const Checkbox = React.forwardRef<
     ref={ref}
     className={cn(
       'fmtm-peer fmtm-h-4 fmtm-w-4 fmtm-shrink-0 fmtm-rounded-sm fmtm-border fmtm-border-[#7A7676] fmtm-shadow focus-visible:fmtm-outline-none focus-visible:fmtm-ring-1 disabled:fmtm-cursor-not-allowed disabled:fmtm-opacity-50  data-[state=checked]:fmtm-text-primary-[#7A7676]',
+      { 'disabled:fmtm-cursor-not-allowed': props.disabled },
       className,
     )}
     {...props}
@@ -30,16 +32,32 @@ const Checkbox = React.forwardRef<
 ));
 Checkbox.displayName = CheckboxPrimitive.Root.displayName;
 
-export const CustomCheckbox = ({ label, checked, onCheckedChange, className, labelClickable }: CustomCheckboxType) => {
-  const labelStyle = labelClickable ? { cursor: 'pointer' } : {};
+export const CustomCheckbox = ({
+  label,
+  checked,
+  onCheckedChange,
+  className,
+  labelClickable,
+  disabled,
+}: CustomCheckboxType) => {
+  const labelStyle = {
+    width: 'calc(100% - 32px)',
+    ...(labelClickable ? { cursor: disabled ? 'not-allowed' : 'pointer' } : {}),
+  };
+
+  const handleLabelClick = () => {
+    if (!disabled && labelClickable) {
+      onCheckedChange(!checked);
+    }
+  };
 
   return (
     <div className="fmtm-flex fmtm-gap-2 sm:fmtm-gap-4">
-      <Checkbox checked={checked} onCheckedChange={onCheckedChange} className="fmtm-mt-[2px]" />
+      <Checkbox checked={checked} onCheckedChange={onCheckedChange} className="fmtm-mt-[2px]" disabled={disabled} />
       <p
-        style={{ width: 'calc(100% - 32px)', ...labelStyle }}
+        style={labelStyle}
         className={`fmtm-text-[#7A7676] fmtm-font-archivo fmtm-text-base fmtm-break-words ${className}`}
-        onClick={() => labelClickable && onCheckedChange(!checked)}
+        onClick={labelClickable && !disabled ? handleLabelClick : undefined}
       >
         {label}
       </p>

src/frontend/src/components/createnewproject/SelectForm.tsx

@@ -135,7 +135,7 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) =>
                 </p>
               </div>
               <CustomCheckbox
-                key="fillODKCredentials"
+                key="uploadCustomXForm"
                 label="Upload a custom XLSForm instead"
                 checked={formValues.formWays === 'custom_form'}
                 onCheckedChange={(status) => {
@@ -146,17 +146,46 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) =>
                   }
                 }}
                 className="fmtm-text-black"
+                labelClickable
+                disabled={!formValues.formCategorySelection}
               />
               {formValues.formWays === 'custom_form' ? (
-                <FileInputComponent
-                  onChange={changeFileHandler}
-                  onResetFile={resetFile}
-                  customFile={customFormFile}
-                  btnText="Select a Form"
-                  accept=".xls,.xlsx,.xml"
-                  fileDescription="*The supported file formats are .xlsx, .xls, .xml"
-                  errorMsg={errors.customFormUpload}
-                />
+                <div>
+                  <p className="fmtm-text-base fmtm-mt-2">
+                    Please extend upon the existing XLSForm for the selected category:
+                  </p>
+                  <p className="fmtm-text-base fmtm-mt-2">
+                    <a
+                      href={`${import.meta.env.VITE_API_URL}/helper/download-template-xlsform?category=${
+                        formValues.formCategorySelection
+                      }`}
+                      target="_"
+                      className="fmtm-text-blue-600 hover:fmtm-text-blue-700 fmtm-cursor-pointer fmtm-underline"
+                    >
+                      Download Form
+                    </a>
+                  </p>
+                  <p className="fmtm-text-base fmtm-mt-2">
+                    <a
+                      href={`https://xlsforms.fmtm.dev/?url=${
+                        import.meta.env.VITE_API_URL
+                      }/helper/download-template-xlsform?category=${formValues.formCategorySelection}`}
+                      target="_"
+                      className="fmtm-text-blue-600 hover:fmtm-text-blue-700 fmtm-cursor-pointer fmtm-underline"
+                    >
+                      Edit Interactively
+                    </a>
+                  </p>
+                  <FileInputComponent
+                    onChange={changeFileHandler}
+                    onResetFile={resetFile}
+                    customFile={customFormFile}
+                    btnText="Select a Form"
+                    accept=".xls,.xlsx,.xml"
+                    fileDescription="*The supported file formats are .xlsx, .xls, .xml"
+                    errorMsg={errors.customFormUpload}
+                  />
+                </div>
               ) : null}
             </div>
             <div className="fmtm-flex fmtm-gap-5 fmtm-mx-auto fmtm-mt-10 fmtm-my-5">