Merge branch 'development' into build/nginx-certbot

.github/workflows/build_ci_img.yml

@@ -22,3 +22,14 @@ jobs:
       build_target: ci
       image_tags: |
         "ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}"
+
+  invalidate-cache:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Delete CI Img Cache
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh extension install actions/gh-actions-cache
+          gh actions-cache delete image-cache-${{ runner.os }} -R \
+            ${{ github.repository }} -B ${{ github.ref_name }} --confirm

.pre-commit-config.yaml

@@ -8,15 +8,15 @@ repos:
 
   # Autoformat: Python code
   - repo: https://github.com/psf/black
-    rev: 23.10.0
+    rev: 23.10.1
     hooks:
       - id: black
         files: ^src/backend/(?:.*/)*.*$
         args: [--target-version=py39]
 
   # Lint / autoformat: Python code
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.1.1"
+    rev: "v0.1.3"
     hooks:
       - id: ruff
         files: ^src/backend/(?:.*/)*.*$

src/backend/app/projects/project_crud.py

@@ -576,6 +576,12 @@ def remove_z_dimension(coord):
 
 
 def get_osm_extracts(boundary: str):
+    """Request an extract from raw-data-api and extract the file contents.
+
+    - The query is posted to raw-data-api and job initiated for fetching the extract.
+    - The status of the job is polled every few seconds, until 'SUCCESS' is returned.
+    - The resulting zip file is downloaded, extracted, and data returned.
+    """
     # Filters for osm extracts
     query = {
         "filters": {
@@ -587,43 +593,54 @@ def get_osm_extracts(boundary: str):
         }
     }
 
+    # Boundary to extract data for
     json_boundary = json.loads(boundary)
-
     if json_boundary.get("features", None) is not None:
         query["geometry"] = json_boundary
         # query["geometry"] = json_boundary["features"][0]["geometry"]
-
     else:
         query["geometry"] = json_boundary
 
+    # Filename to generate
+    query["fileName"] = "extract"
+    # File format to generate
+    query["outputType"] = "geojson"
+    extract_filename = f'{query["fileName"]}.{query["outputType"]}'
+    log.debug(f"Setting data extract file name to: {extract_filename}")
+
+    log.debug(f"Query for raw data api: {query}")
     base_url = settings.UNDERPASS_API_URL
     query_url = f"{base_url}/snapshot/"
     headers = {"accept": "application/json", "Content-Type": "application/json"}
 
+    # Send the request to raw data api
     result = requests.post(query_url, data=json.dumps(query), headers=headers)
 
     if result.status_code == 200:
         task_id = result.json()["task_id"]
     else:
         return False
 
+    # Check status of task (PENDING, or SUCCESS)
     task_url = f"{base_url}/tasks/status/{task_id}"
-    # extracts = requests.get(task_url)
     while True:
         result = requests.get(task_url, headers=headers)
         if result.json()["status"] == "PENDING":
-            time.sleep(1)
+            # Wait 2 seconds before polling again
+            time.sleep(2)
         elif result.json()["status"] == "SUCCESS":
             break
 
+    # TODO update code to generate fgb file format
+    # then input the download_url directly into our database
+    # (no need to download the file and extract)
     zip_url = result.json()["result"]["download_url"]
-    zip_url
     result = requests.get(zip_url, headers=headers)
     # result.content
     fp = BytesIO(result.content)
     zfp = zipfile.ZipFile(fp, "r")
-    zfp.extract("Export.geojson", "/tmp/")
-    data = json.loads(zfp.read("Export.geojson"))
+    zfp.extract(extract_filename, "/tmp/")
+    data = json.loads(zfp.read(extract_filename))
 
     for feature in data["features"]:
         properties = feature["properties"]
@@ -721,6 +738,10 @@ def split_polygon_into_tasks(
     db.add(db_task)
     db.commit()
 
+    # Get the data extract from raw-data-api
+    # Input into DbBuildings and DbOsmLines
+    # TODO update to use flatgeobuf file directly
+    # No need to store in our database
     if not has_data_extracts:
         data = get_osm_extracts(json.dumps(boundary_data))
         if not data:
@@ -1503,9 +1524,13 @@ def generate_appuser_files(
                 boundary = {"type": "Feature", "properties": {}, "geometry": outline}
                 data_extract = pg.execQuery(boundary)
                 filter = FilterData(xlsform)
-                filtered_data_extract = filter.cleanData(data_extract)
 
                 updated_data_extract = {"type": "FeatureCollection", "features": []}
+                filtered_data_extract = (
+                    filter.cleanData(data_extract)
+                    if data_extract
+                    else updated_data_extract
+                )
 
                 # Collect feature mappings for bulk insert
                 feature_mappings = []
@@ -1655,7 +1680,7 @@ def get_task_geometry(db: Session, project_id: int):
 
 
 async def get_project_features_geojson(db: Session, project_id: int):
-    # Get the geojson of those features for this task.
+    """Get a geojson of all features for a task."""
     query = text(
         f"""SELECT jsonb_build_object(
                 'type', 'FeatureCollection',

src/backend/app/projects/project_routes.py

@@ -673,16 +673,16 @@ def get_project_features(
     task_id: int = None,
     db: Session = Depends(database.get_db),
 ):
-    """Get api for fetching all the features of a project.
+    """Fetch all the features for a project.
 
-    This endpoint allows you to get all the features of a project.
+    The features are generated from raw-data-api.
 
-    ## Request Body
-    - `project_id` (int): the project's id. Required.
-
-    ## Response
-    - Returns a JSON object containing a list of features.
+    Args:
+        project_id (int): The project id.
+        task_id (int): The task id.
 
+    Returns:
+        feature(json): JSON object containing a list of features
     """
     features = project_crud.get_project_features(db, project_id, task_id)
     return features

src/backend/app/projects/project_schemas.py

@@ -57,8 +57,6 @@ class BETAProjectUpload(BaseModel):
 
 class Feature(BaseModel):
     id: int
-    project_id: int
-    task_id: Optional[int] = None
     geometry: Optional[GeojsonFeature] = None
 
 

src/frontend/src/components/ProjectDetails/ProjectOptions.tsx

@@ -3,6 +3,7 @@ import CoreModules from '../../shared/CoreModules';
 import AssetModules from '../../shared/AssetModules';
 import environment from '../../environment';
 import { DownloadDataExtract, DownloadProjectForm } from '../../api/Project';
+import { ProjectActions } from '../../store/slices/ProjectSlice';
 
 const ProjectOptions = ({ setToggleGenerateModal }) => {
   const dispatch = CoreModules.useAppDispatch();
@@ -119,7 +120,10 @@ const ProjectOptions = ({ setToggleGenerateModal }) => {
             </CoreModules.Button>
           </CoreModules.Link>
           <CoreModules.Button
-            onClick={() => setToggleGenerateModal(true)}
+            onClick={() => {
+              setToggleGenerateModal(true);
+              dispatch(ProjectActions.SetMobileFooterSelection('explore'));
+            }}
             variant="contained"
             color="error"
             sx={{ width: '200px', mr: '15px' }}

src/frontend/src/components/common/Select.tsx

@@ -131,10 +131,10 @@ export const CustomSelect = ({
   errorMsg,
 }: ICustomSelect) => {
   return (
-    <div className="">
+    <div className="fmtm-w-full">
       <p className="fmtm-text-[1rem] fmtm-mb-2 fmtm-font-semibold ">{title}</p>
       <div className="fmtm-flex fmtm-items-end ">
-        <div className="fmtm-w-[14rem] ">
+        <div className="fmtm-w-full">
           <Select value={value} onValueChange={(value) => onValueChange(value)}>
             <SelectTrigger className="">
               <SelectValue placeholder={placeholder} />

src/frontend/src/components/createnewproject/ProjectDetailsForm.tsx

@@ -166,8 +166,8 @@ const ProjectDetailsForm = ({ flag }) => {
             </div>
           </div>
           <div className="md:fmtm-w-[50%] fmtm-flex fmtm-flex-col fmtm-gap-6">
-            <div>
-              <div className="fmtm-flex fmtm-items-center ">
+            <div className="">
+              <div className="fmtm-flex fmtm-items-center fmtm-max-w-[18rem]">
                 <CustomSelect
                   title="Organization Name"
                   placeholder="Organization Name"
@@ -179,7 +179,7 @@ const ProjectDetailsForm = ({ flag }) => {
                   onValueChange={(value) => handleCustomChange('organisation_id', value && +value)}
                 />
                 <AssetModules.AddIcon
-                  className="fmtm-bg-red-600 fmtm-text-white fmtm-rounded-full fmtm-mb-[0.15rem] hover:fmtm-bg-red-700 hover:fmtm-cursor-pointer fmtm-ml-5 fmtm-mt-4"
+                  className="fmtm-bg-red-600 fmtm-text-white fmtm-rounded-full fmtm-mb-[0.15rem] hover:fmtm-bg-red-700 hover:fmtm-cursor-pointer fmtm-ml-5 fmtm-mt-9"
                   onClick={() => createPopup('Create Organization', 'createOrganization?popup=true')}
                 />
               </div>

src/frontend/src/components/createnewproject/SelectForm.tsx

@@ -85,7 +85,7 @@ const SelectForm = ({ flag, geojsonFile, customFormFile, setCustomFormFile }) =>
             onSubmit={handleSubmit}
             className="fmtm-flex fmtm-flex-col fmtm-gap-6 lg:fmtm-w-[40%] fmtm-justify-between"
           >
-            <div className="">
+            <div className="fmtm-w-[13.35rem]">
               <CustomSelect
                 title="Select form category"
                 placeholder="Select form category"

src/frontend/src/components/createnewproject/SplitTasks.tsx

@@ -94,7 +94,7 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo
             odk_central_password: projectDetails.odk_central_password,
           },
           // dont send xform_title if upload custom form is selected
-          xform_title: projectDetails.formWays === 'custom_form' ? null : projectDetails.formCategorySelection,
+          xform_title: projectDetails.formCategorySelection,
           dimension: projectDetails.dimension,
           splitting_algorithm: splitTasksSelection,
           form_ways: projectDetails.formWays,
@@ -136,10 +136,10 @@ const SplitTasks = ({ flag, geojsonFile, setGeojsonFile, customLineUpload, custo
         }),
       );
     } else if (splitTasksSelection === 'task_splitting_algorithm') {
-      const a = document.createElement('a');
-      a.href = URL.createObjectURL(drawnGeojsonFile);
-      a.download = 'test.json';
-      a.click();
+      // const a = document.createElement('a');
+      // a.href = URL.createObjectURL(drawnGeojsonFile);
+      // a.download = 'test.json';
+      // a.click();
       dispatch(
         TaskSplittingPreviewService(
           `${import.meta.env.VITE_API_URL}/projects/task_split`,

src/frontend/src/components/home/HomePageFilters.tsx

@@ -130,13 +130,6 @@ const HomePageFilters = ({ onSearch, filteredProjectCount, totalProjectCount })
               ></input>
               <i className="material-icons">search</i>
             </div>
-            <AssetModules.FilterAltIcon
-              sx={{
-                fontSize: 25,
-                color: '#9B9999',
-                cursor: 'pointer',
-              }}
-            />
           </div>
           <div className="fmtm-flex fmtm-items-center fmtm-gap-2">
             <p>Show Map</p>
@@ -148,7 +141,7 @@ const HomePageFilters = ({ onSearch, filteredProjectCount, totalProjectCount })
         </div>
         <div className="fmtm-mt-6 fmtm-mb-1 fmtm-flex fmtm-items-center fmtm-justify-between">
           <p className="fmtm-text-[#A8A6A6]">
-            showing {filteredProjectCount} of {totalProjectCount} projects
+            Showing {filteredProjectCount} {filteredProjectCount > 1 ? 'projects' : 'project'}
           </p>
           {/* <div className="fmtm-flex fmtm-gap-1">
             <AssetModules.WindowIcon

src/frontend/src/utilities/CustomDrawer.jsx

@@ -3,9 +3,13 @@ import SwipeableDrawer from '@mui/material/SwipeableDrawer';
 import CoreModules from '../shared/CoreModules';
 import AssetModules from '../shared/AssetModules';
 import { NavLink } from 'react-router-dom';
+import { createLoginWindow } from '../utilfunctions/login';
+import { LoginActions } from '../store/slices/LoginSlice';
+import { ProjectActions } from '../store/slices/ProjectSlice';
 
-export default function CustomDrawer({ open, placement, size, type, onClose, onSignOut }) {
+export default function CustomDrawer({ open, placement, size, type, onClose, onSignOut, setOpen }) {
   const defaultTheme = CoreModules.useAppSelector((state) => state.theme.hotTheme);
+  const dispatch = CoreModules.useAppDispatch();
 
   const onMouseEnter = (event) => {
     const element = document.getElementById(`text${event.target.id}`);
@@ -81,6 +85,12 @@ export default function CustomDrawer({ open, placement, size, type, onClose, onS
     },
   ];
 
+  const handleOnSignOut = () => {
+    setOpen(false);
+    dispatch(LoginActions.signOut(null));
+    dispatch(ProjectActions.clearProjects([]));
+  };
+
   return (
     <div>
       <React.Fragment>
@@ -144,14 +154,11 @@ export default function CustomDrawer({ open, placement, size, type, onClose, onS
                   <NavLink
                     key={index}
                     to={menuDetails.ref}
-                    style={{
-                      textDecoration: 'inherit',
-                      color: 'inherit',
-                      fontFamily: 'inherit',
-                      fontSize: '1.01587rem',
-                      background: '#000000',
-                      opacity: 0.8,
-                    }}
+                    className={`fmtm-no-underline fmtm-text-inherit fmtm-opacity-80 ${
+                      menuDetails.name === 'Explore Projects' || menuDetails.name === 'Manage Organizations'
+                        ? 'lg:fmtm-hidden'
+                        : ''
+                    }`}
                   >
                     <CoreModules.ListItem
                       id={index.toString()}
@@ -164,6 +171,23 @@ export default function CustomDrawer({ open, placement, size, type, onClose, onS
                   </NavLink>
                 ),
               )}
+              <div className="fmtm-ml-4 fmtm-mt-2 lg:fmtm-hidden">
+                {token != null ? (
+                  <div
+                    className="fmtm-text-[#d73e3e] hover:fmtm-text-[#d73e3e] fmtm-cursor-pointer fmtm-opacity-80"
+                    onClick={handleOnSignOut}
+                  >
+                    Sign Out
+                  </div>
+                ) : (
+                  <div
+                    className="fmtm-text-[#44546a] hover:fmtm-text-[#d73e3e] fmtm-cursor-pointer fmtm-opacity-80"
+                    onClick={() => createLoginWindow('/')}
+                  >
+                    Sign In
+                  </div>
+                )}
+              </div>
             </CoreModules.List>
           </CoreModules.Stack>
         </SwipeableDrawer>

src/frontend/src/utilities/PrimaryAppBar.tsx

@@ -66,6 +66,7 @@ export default function PrimaryAppBar() {
         size={windowSize}
         type={type}
         onSignOut={handleOnSignOut}
+        setOpen={setOpen}
       />
       <CoreModules.AppBar
         position="static"

src/frontend/src/views/NewProjectDetails.jsx

@@ -186,6 +186,12 @@ const Home = () => {
     fillOpacity: '0',
   };
 
+  useEffect(() => {
+    if (mobileFooterSelection !== 'explore') {
+      setToggleGenerateModal(false);
+    }
+  }, [mobileFooterSelection]);
+
   return (
     <div>
       {/* Customized Modal For Generate Tiles */}