build: added packages python-calamine and openpyxl (#291)

* feat: added packages python-calamine and xlsxwriter

* feat: updated xls form to include building_exists condition

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* build: replace xlsxwriter with openpyxl + relock deps

* refactor: replace xlsxwriter usage with openpyxl

* test: start simple test to check if adding mandatory fields works

* build: tweak mandatory fields + digitization fields forms

* fix: apply monkeypatch for usage of calamine xlsx reader

* fix: append or overwrite mandatory entities sheet (do not concat)

* test: add tests for appending mandatory fields

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: spwoodcock <[email protected]>

osm_fieldwork/update_form.py

@@ -1,9 +1,13 @@
 from io import BytesIO
 
 import pandas as pd
+from python_calamine.pandas import pandas_monkeypatch
 
 from osm_fieldwork.xlsforms import xlsforms_path
 
+# Monkeypatch pandas to add calamine driver
+pandas_monkeypatch()
+
 
 def merge_sheets(mandatory_df, custom_df, digitisation_df):
     # Remove rows with None in 'name' column
@@ -29,9 +33,23 @@ def merge_sheets(mandatory_df, custom_df, digitisation_df):
     mandatory_df_filtered = mandatory_df[~mandatory_df["name"].isin(common_fields)]
     digitisation_df_filtered = digitisation_df[~digitisation_df["name"].isin(common_fields)]
 
+    group_row = pd.DataFrame(
+        {
+            "type": ["begin group"],
+            "name": ["survey_questions"],
+            "label": ["Survey Form"],
+            "relevant": [
+                "${building_exists} = 'yes'"
+            ],  # Add the relevant condition to display this group only if "Yes" is selected
+        }
+    )
+
+    end_group_row = pd.DataFrame({"type": ["end group"], "name": ["end_survey_questions"], "label": ["End Survey Form"]})
+
     # Concatenate: mandatory fields at the top, custom common fields, remaining custom fields, and finally append form fields
     merged_df = pd.concat(
-        [custom_common_df, mandatory_df_filtered, custom_non_common_df, digitisation_df_filtered], ignore_index=True
+        [custom_common_df, mandatory_df_filtered, group_row, custom_non_common_df, digitisation_df_filtered, end_group_row],
+        ignore_index=True,
     )
 
     return merged_df
@@ -54,17 +72,12 @@ def update_xls_form(custom_form: BytesIO) -> BytesIO:
             mandatory_sheets["choices"], custom_sheets["choices"], digitisation_sheets["choices"]
         )
 
-    # Handle the 'entities' sheet: append or create if not present in custom form
+    # Append or overwrite the existing entities sheet
     if "entities" in mandatory_sheets:
-        if "entities" in custom_sheets:
-            custom_sheets["entities"] = pd.concat(
-                [custom_sheets["entities"], mandatory_sheets["entities"]], ignore_index=True
-            ).drop_duplicates()
-        else:
-            custom_sheets["entities"] = mandatory_sheets["entities"]
+        custom_sheets["entities"] = mandatory_sheets["entities"]
 
     output = BytesIO()
-    with pd.ExcelWriter(output, engine="xlsxwriter") as writer:
+    with pd.ExcelWriter(output, engine="openpyxl") as writer:
         for sheet_name, df in custom_sheets.items():
             df.to_excel(writer, sheet_name=sheet_name, index=False)
 

pyproject.toml

@@ -21,11 +21,13 @@ dependencies = [
     "mercantile>=1.2.1",
     "pySmartDL>=1.3.4",
     "pandas>=1.5.0",
+    "python-calamine>=0.2.3",
+    "openpyxl>=3.0.10",
     "py-cpuinfo>=9.0.0",
     "requests>=2.26.0",
     "pmtiles>=3.2.0",
+    "aiohttp>=3.8.4",
     "osm-rawdata>=0.1.7",
-    "aiohttp>=3.9.3",
 ]
 requires-python = ">=3.10"
 readme = "README.md"