From 122d73541d5ee6bf5e0062e4d14b6f7965288ef4 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 14:59:16 +0545 Subject: [PATCH 01/13] updated summary schemas to use Optional --- src/backend/app/projects/project_schemas.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 0ec20a2709..9d4044a084 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -16,7 +16,7 @@ # along with FMTM. If not, see . # -from typing import List, Union +from typing import List, Union, Optional from geojson_pydantic import Feature from pydantic import BaseModel @@ -59,17 +59,17 @@ class ProjectSummary(BaseModel): id: int = -1 priority: ProjectPriority = ProjectPriority.MEDIUM priority_str: str = priority.name - title: str = None - location_str: str = None - description: str = None - num_contributors: int = None - total_tasks: int = None - tasks_mapped: int = None - tasks_validated: int = None - tasks_bad: int = None - hashtags: List[str] = None - organisation_id: int = None - organisation_logo: str = None + title: Optional[str] + location_str: Optional[str] + description: Optional[str] + total_tasks: Optional[int] + tasks_mapped: Optional[int] + num_contributors: Optional[int] + tasks_validated: Optional[int] + tasks_bad: Optional[int] + hashtags: Optional[List[str]] + organisation_id: Optional[int] + organisation_logo: Optional[str] class ProjectBase(BaseModel): From 58212eb5000d5ebf60e8c06ace19ad1c14d677c6 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 15:01:04 +0545 Subject: [PATCH 02/13] updated get organisation by name api --- .../app/organization/organization_crud.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/backend/app/organization/organization_crud.py b/src/backend/app/organization/organization_crud.py index 38159ed89f..df297dd14b 100644 --- a/src/backend/app/organization/organization_crud.py +++ b/src/backend/app/organization/organization_crud.py @@ -22,7 +22,7 @@ import string from fastapi import HTTPException, File,UploadFile import re - +from sqlalchemy import func from sqlalchemy.orm import Session from ..db import db_models @@ -44,15 +44,12 @@ def generate_slug(text: str) -> str: async def get_organisation_by_name(db: Session, name: str): - - # Construct the SQL query with the case-insensitive search - query = f"SELECT * FROM organisations WHERE LOWER(name) LIKE LOWER('%{name}%') LIMIT 1" - - # Execute the query and retrieve the result - result = db.execute(query) - - # Fetch the first row of the result - db_organisation = result.fetchone() + # Use SQLAlchemy's query-building capabilities + db_organisation = ( + db.query(db_models.DbOrganisation) + .filter(func.lower(db_models.DbOrganisation.name).like(func.lower(f'%{name}%'))) + .first() + ) return db_organisation From ef0f63d3a59667de9590fd2dbc3c6461b6438297 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 15:01:28 +0545 Subject: [PATCH 03/13] fix: import error in json2osm --- src/backend/app/projects/project_crud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index d02f0979d7..b0e95963ca 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -44,7 +44,7 @@ from osm_fieldwork.make_data_extract import PostgresClient from osm_fieldwork.OdkCentral import OdkAppUser from osm_fieldwork.xlsforms import xlsforms_path -from osm_fieldwork.json2osm import json2osm +from osm_fieldwork import json2osm from shapely import wkt from shapely.geometry import MultiPolygon, Polygon, mapping, shape from sqlalchemy import and_, column, func, inspect, select, table From 6deb2eb08f88051cbd7d7027eda95f3f7494aeea Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 15:06:22 +0545 Subject: [PATCH 04/13] fix: list forms api --- src/backend/app/central/central_crud.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/backend/app/central/central_crud.py b/src/backend/app/central/central_crud.py index f28e506021..84953e9e38 100644 --- a/src/backend/app/central/central_crud.py +++ b/src/backend/app/central/central_crud.py @@ -330,12 +330,23 @@ def list_submissions(project_id: int, odk_central: project_schemas.ODKCentral = def get_form_list(db: Session, skip: int, limit: int): """Returns the list of id and title of xforms from the database.""" try: - return ( + forms = ( db.query(db_models.DbXForm.id, db_models.DbXForm.title) .offset(skip) .limit(limit) .all() ) + + result_dict = [] + for form in forms: + form_dict = { + 'id': form[0], # Assuming the first element is the ID + 'title': form[1] # Assuming the second element is the title + } + result_dict.append(form_dict) + + return result_dict + except Exception as e: log.error(e) raise HTTPException(e) from e From c4dd3411a366fad1f27be416f288579463b8d29d Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 15:45:39 +0545 Subject: [PATCH 05/13] updated schemas --- src/backend/app/projects/project_schemas.py | 24 +++++++++++---------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 9d4044a084..eef1611df8 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -50,11 +50,18 @@ class BETAProjectUpload(BaseModel): xform_title: Union[str, None] odk_central: ODKCentral hashtags: Union[List[str], None] - organisation_id: int = None + organisation_id: Optional[int] # city: str # country: str +class Feature(BaseModel): + id: int + project_id: int + task_id: Optional[int] + geometry: Optional[Feature] + + class ProjectSummary(BaseModel): id: int = -1 priority: ProjectPriority = ProjectPriority.MEDIUM @@ -79,11 +86,11 @@ class ProjectBase(BaseModel): project_info: List[ProjectInfo] status: ProjectStatus # location_str: str - outline_geojson: Feature = None - project_tasks: List[tasks_schemas.Task] = None - xform_title: str = None - hashtags: List[str] = None - organisation_id: int = None + # outline_geojson: Optional[Feature] + project_tasks: Optional[List[tasks_schemas.Task]] + xform_title: Optional[str] + hashtags: Optional[List[str]] + organisation_id: Optional[int] class ProjectOut(ProjectBase): @@ -91,8 +98,3 @@ class ProjectOut(ProjectBase): -class Feature(BaseModel): - id: int - project_id: int - task_id: int = None - geometry: Feature From 1074137ccd557c3afdc84781c14356663ee47e68 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 16:37:15 +0545 Subject: [PATCH 06/13] update raw query issue in upload multipolygon --- src/backend/app/projects/project_crud.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index b0e95963ca..c40db85453 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -50,6 +50,7 @@ from sqlalchemy import and_, column, func, inspect, select, table from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session +from sqlalchemy import text from sqlalchemy.sql import text from cpuinfo import get_cpu_info from ..db import database @@ -436,14 +437,13 @@ def remove_z_dimension(coord): ) db.commit() + + # Generate project outline from tasks - # query = f'''SELECT ST_AsText(ST_Buffer(ST_Union(outline), 0.5, 'endcap=round')) as oval_envelope - # FROM tasks - # where project_id={project_id};''' + query = text(f"""SELECT ST_AsText(ST_ConvexHull(ST_Collect(outline))) + FROM tasks + WHERE project_id={project_id};""") - query = f"""SELECT ST_AsText(ST_ConvexHull(ST_Collect(outline))) - FROM tasks - WHERE project_id={project_id};""" log.debug("Generating project outline from tasks") result = db.execute(query) data = result.fetchone() From c430ea91e81695c404537d333f67eb14eed8d759 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 16:49:00 +0545 Subject: [PATCH 07/13] updated tasks list function --- src/backend/app/tasks/tasks_crud.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/backend/app/tasks/tasks_crud.py b/src/backend/app/tasks/tasks_crud.py index fd4066945c..a1cc26c9e3 100644 --- a/src/backend/app/tasks/tasks_crud.py +++ b/src/backend/app/tasks/tasks_crud.py @@ -50,10 +50,18 @@ async def get_task_count_in_project(db: Session, project_id: int): def get_task_lists(db: Session, project_id: int): """Get a list of tasks for a project.""" - query = f"""select id from tasks where project_id = {project_id}""" - result = db.execute(query) - tasks = [task[0] for task in result.fetchall()] - return tasks + query = text(""" + SELECT id + FROM tasks + WHERE project_id = :project_id + """) + + # Then execute the query with the desired parameter + result = db.execute(query, {"project_id": project_id}) + + # Fetch the result + task_ids = [row.id for row in result] + return task_ids def get_tasks( From b07e8141955283d48d99fe64fa631324ecc7c57e Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 17:05:02 +0545 Subject: [PATCH 08/13] updated raw sql query in generate app user files --- src/backend/app/projects/project_crud.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index c40db85453..143f72f2d7 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -1207,7 +1207,7 @@ def generate_task_files( # Get the features for this task. # Postgis query to filter task inside this task outline and of this project # Update those features and set task_id - query = f"""UPDATE features + query = text(f"""UPDATE features SET task_id={task_id} WHERE id IN ( SELECT id @@ -1216,12 +1216,12 @@ def generate_task_files( AND ST_IsValid(geometry) AND ST_IsValid('{task.outline}'::Geometry) AND ST_Contains('{task.outline}'::Geometry, ST_Centroid(geometry)) - )""" + )""") result = db.execute(query) # Get the geojson of those features for this task. - query = f"""SELECT jsonb_build_object( + query = text(f"""SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', jsonb_agg(feature) ) @@ -1234,9 +1234,10 @@ def generate_task_files( ) AS feature FROM features WHERE project_id={project_id} and task_id={task_id} - ) features;""" + ) features;""") result = db.execute(query) + features = result.fetchone()[0] upload_media = False if features['features'] is None else True From 508cd9f6a1b10bbaa80386f98955c81587397232 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 17:10:34 +0545 Subject: [PATCH 09/13] update raw query call process in point on surface api --- src/backend/app/tasks/tasks_routes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/app/tasks/tasks_routes.py b/src/backend/app/tasks/tasks_routes.py index e97a3dbed3..970e0df6e3 100644 --- a/src/backend/app/tasks/tasks_routes.py +++ b/src/backend/app/tasks/tasks_routes.py @@ -29,6 +29,7 @@ from . import tasks_crud, tasks_schemas from ..projects import project_crud, project_schemas from ..central import central_crud +from sqlalchemy.sql import text router = APIRouter( @@ -89,12 +90,11 @@ async def get_point_on_surface( List[Tuple[int, str]]: A list of tuples containing the task ID and the centroid as a string. """ - query = f""" + query = text(f""" SELECT id, ARRAY_AGG(ARRAY[ST_X(ST_PointOnSurface(outline)), ST_Y(ST_PointOnSurface(outline))]) AS point FROM tasks WHERE project_id = {project_id} - GROUP BY id; - """ + GROUP BY id; """) result = db.execute(query) result = result.fetchall() From a5f6dd6aa978c21b994e77e40ae11cf029c1e657 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Tue, 5 Sep 2023 17:16:02 +0545 Subject: [PATCH 10/13] point on surface api updated for result in a dict format --- src/backend/app/tasks/tasks_routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/backend/app/tasks/tasks_routes.py b/src/backend/app/tasks/tasks_routes.py index 970e0df6e3..ccc9f0170c 100644 --- a/src/backend/app/tasks/tasks_routes.py +++ b/src/backend/app/tasks/tasks_routes.py @@ -97,8 +97,8 @@ async def get_point_on_surface( GROUP BY id; """) result = db.execute(query) - result = result.fetchall() - return result + result_dict_list = [{"id": row[0], "point": row[1]} for row in result.fetchall()] + return result_dict_list @router.post("/near_me", response_model=tasks_schemas.TaskOut) From 0f5014db5fc8792ebbb2071cd39fdbdae830042f Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Wed, 6 Sep 2023 12:56:22 +0545 Subject: [PATCH 11/13] fix:query in tasks-features --- src/backend/app/tasks/tasks_routes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/app/tasks/tasks_routes.py b/src/backend/app/tasks/tasks_routes.py index ccc9f0170c..f6e08d851a 100644 --- a/src/backend/app/tasks/tasks_routes.py +++ b/src/backend/app/tasks/tasks_routes.py @@ -178,9 +178,9 @@ async def task_features_count( ) def process_task(task): - feature_count_query = f""" - select count(*) from features where project_id = {project_id} and task_id = {task} - """ + feature_count_query = text(f""" + select count(*)from features where project_id = {project_id} and task_id = {task} + """) result = db.execute(feature_count_query) feature_count = result.fetchone() @@ -190,7 +190,7 @@ def process_task(task): # form_details = central_crud.get_form_full_details(project.odkid, task, odk_credentials) return { "task_id": task, - "feature_count": feature_count["count"], + "feature_count": feature_count[0], # 'submission_count': form_details['submissions'], "submission_count": len(submission_list) if isinstance(submission_list, list) From 0a2e52c5d3d97ef0b5824263718b00d39a0f6b62 Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Wed, 6 Sep 2023 13:38:20 +0545 Subject: [PATCH 12/13] Optional field in schemas updated with None value in default --- src/backend/app/projects/project_schemas.py | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index eef1611df8..77b60c662d 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -50,7 +50,7 @@ class BETAProjectUpload(BaseModel): xform_title: Union[str, None] odk_central: ODKCentral hashtags: Union[List[str], None] - organisation_id: Optional[int] + organisation_id: Optional[int] = None # city: str # country: str @@ -58,25 +58,25 @@ class BETAProjectUpload(BaseModel): class Feature(BaseModel): id: int project_id: int - task_id: Optional[int] - geometry: Optional[Feature] + task_id: Optional[int] = None + geometry: Optional[Feature] = None class ProjectSummary(BaseModel): id: int = -1 priority: ProjectPriority = ProjectPriority.MEDIUM priority_str: str = priority.name - title: Optional[str] - location_str: Optional[str] - description: Optional[str] - total_tasks: Optional[int] - tasks_mapped: Optional[int] - num_contributors: Optional[int] - tasks_validated: Optional[int] - tasks_bad: Optional[int] - hashtags: Optional[List[str]] - organisation_id: Optional[int] - organisation_logo: Optional[str] + title: Optional[str] = None + location_str: Optional[str] = None + description: Optional[str] = None + total_tasks: Optional[int] = None + tasks_mapped: Optional[int] = None + num_contributors: Optional[int] = None + tasks_validated: Optional[int] = None + tasks_bad: Optional[int] = None + hashtags: Optional[List[str]] = None + organisation_id: Optional[int] = None + organisation_logo: Optional[str] = None class ProjectBase(BaseModel): @@ -88,9 +88,9 @@ class ProjectBase(BaseModel): # location_str: str # outline_geojson: Optional[Feature] project_tasks: Optional[List[tasks_schemas.Task]] - xform_title: Optional[str] - hashtags: Optional[List[str]] - organisation_id: Optional[int] + xform_title: Optional[str] = None + hashtags: Optional[List[str]] = None + organisation_id: Optional[int] = None class ProjectOut(ProjectBase): From b44be94559f52be26169f29a66ec24fe78ff923b Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Wed, 6 Sep 2023 17:24:27 +0545 Subject: [PATCH 13/13] fix: task split algorithm --- src/backend/app/projects/project_crud.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 143f72f2d7..8be5db4f81 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -700,15 +700,15 @@ def process_polygon(db:Session, project_id:uuid.UUID, boundary_data:str, no_of_b db.commit() else: # Remove the polygons outside of the project AOI using a parameterized query - query = f""" + query = text(f""" DELETE FROM ways_poly WHERE NOT ST_Within(ST_Centroid(ways_poly.geom), (SELECT geom FROM project_aoi WHERE project_id = '{project_id}')); - """ + """) result = db.execute(query) db.commit() with open('app/db/split_algorithm.sql', 'r') as sql_file: query = sql_file.read() - result = db.execute(query, params={'num_buildings': no_of_buildings}) + result = db.execute(text(query), params={'num_buildings': no_of_buildings}) result = result.fetchall() db.query(db_models.DbBuildings).delete() db.query(db_models.DbOsmLines).delete()