Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Aug 8, 2024
1 parent 23331d3 commit c284271
Show file tree
Hide file tree
Showing 5 changed files with 49 additions and 42 deletions.
2 changes: 1 addition & 1 deletion src/backend/app/db/db_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -562,4 +562,4 @@ class DbSubmissionPhotos(Base):
project_id = cast(int, Column(Integer))
task_id = cast(int, Column(Integer))
submission_id = cast(str, Column(String))
s3_path = cast(str, Column(String))
s3_path = cast(str, Column(String))
8 changes: 4 additions & 4 deletions src/backend/app/s3.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
"""Initialise the S3 buckets for FMTM to function."""

from fastapi import HTTPException
import json
import sys
from io import BytesIO
from typing import Any

from fastapi import HTTPException
from loguru import logger as log
from minio import Minio
from minio.commonconfig import CopySource
Expand Down Expand Up @@ -40,6 +40,7 @@ def s3_client():
# print(creds.access_key)
# print(creds.secret_key)


def object_exists(bucket_name: str, s3_path: str) -> bool:
"""Check if an object exists in an S3 bucket using stat_object.
Expand All @@ -57,13 +58,12 @@ def object_exists(bucket_name: str, s3_path: str) -> bool:
client.stat_object(bucket_name, s3_path)
return True
except S3Error as e:
if e.code == 'NoSuchKey':
if e.code == "NoSuchKey":
return False
else:
# Handle other exceptions
raise HTTPException(
status_code = HTTPStatus.BAD_REQUEST,
detail = str(e)
status_code=HTTPStatus.BAD_REQUEST, detail=str(e)
) from e


Expand Down
61 changes: 33 additions & 28 deletions src/backend/app/submissions/submission_crud.py
Original file line number Diff line number Diff line change
Expand Up @@ -569,13 +569,9 @@ async def get_submission_detail(


async def upload_attachment_to_s3(
project_id: int,
instance_ids: list,
background_task_id: uuid.UUID,
db: Session
):
"""
Uploads attachments to S3 for a given project and instance IDs.
project_id: int, instance_ids: list, background_task_id: uuid.UUID, db: Session
):
"""Uploads attachments to S3 for a given project and instance IDs.
Args:
project_id (int): The ID of the project.
Expand All @@ -590,7 +586,7 @@ async def upload_attachment_to_s3(
Exception: If an error occurs during the upload process.
"""
try:
project = await project_deps.get_project_by_id(db,project_id)
project = await project_deps.get_project_by_id(db, project_id)
db_xform = await project_deps.get_project_xform(db, project_id)
odk_central = await project_deps.get_odk_credentials(db, project_id)
xform = get_odk_form(odk_central)
Expand All @@ -599,29 +595,33 @@ async def upload_attachment_to_s3(
for instance_id in instance_ids:
submission_detail = await get_submission_detail(instance_id, project, db)
attachments = submission_detail["verification"]["image"]

if not isinstance(attachments, list):
attachments = [attachments]

for idx, filename in enumerate(attachments):
s3_key = f"fmtm-data/{project.organisation_id}/{project_id}/{instance_id}/{idx+1}.jpeg"

if object_exists(s3_bucket, s3_key):
log.warning(f"Object {s3_key} already exists in S3. Skipping upload.")
log.warning(
f"Object {s3_key} already exists in S3. Skipping upload."
)
continue

try:
if attachment:= xform.getMedia(
project.odkid,
str(instance_id),
db_xform.odk_form_id,
str(filename)
if attachment := xform.getMedia(
project.odkid,
str(instance_id),
db_xform.odk_form_id,
str(filename),
):
# Convert the attachment to a BytesIO stream
image_stream = io.BytesIO(attachment)

# Upload the attachment to S3
add_obj_to_bucket(s3_bucket, image_stream, s3_key, content_type="image/jpeg")
add_obj_to_bucket(
s3_bucket, image_stream, s3_key, content_type="image/jpeg"
)

# Generate the image URL
img_url = f"{settings.S3_DOWNLOAD_ROOT}/{settings.S3_BUCKET_NAME}/{s3_key}"
Expand All @@ -631,17 +631,22 @@ async def upload_attachment_to_s3(
INSERT INTO submission_photos (project_id, task_id, submission_id, s3_path)
VALUES (:project_id, :task_id, :submission_id, :s3_path)
""")
db.execute(sql, {
"project_id": project_id,
"task_id": submission_detail["task_id"],
"submission_id": instance_id,
"s3_path": img_url
})

db.execute(
sql,
{
"project_id": project_id,
"task_id": submission_detail["task_id"],
"submission_id": instance_id,
"s3_path": img_url,
},
)

except Exception as e:
log.warning(f"Failed to process {filename} for instance {instance_id}: {e}")
log.warning(
f"Failed to process {filename} for instance {instance_id}: {e}"
)
continue

db.commit()
return True

Expand All @@ -651,4 +656,4 @@ async def upload_attachment_to_s3(
update_bg_task_sync = async_to_sync(
project_crud.update_background_task_status_in_database
)
update_bg_task_sync(db, background_task_id, 2, str(e))
update_bg_task_sync(db, background_task_id, 2, str(e))
12 changes: 8 additions & 4 deletions src/backend/app/submissions/submission_routes.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,13 @@
import json
import uuid
from io import BytesIO
from loguru import logger as log
from typing import Annotated, Optional

import geojson
from fastapi import APIRouter, Depends, HTTPException, Query, BackgroundTasks
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query
from fastapi.concurrency import run_in_threadpool
from fastapi.responses import FileResponse, JSONResponse, Response
from loguru import logger as log
from sqlalchemy.orm import Session

from app.auth.auth_schemas import AuthUser, ProjectUserDict
Expand Down Expand Up @@ -387,14 +387,18 @@ async def submission_table(
for submission in submissions:
if submission["__system"]["attachmentsPresent"] != 0:
instance_ids.append(submission["__id"])

if instance_ids:
background_task_id = await project_crud.insert_background_task_into_database(
db, "upload_submission_photos", project.id
)
log.info("uploading submission photos to s3")
background_tasks.add_task(
submission_crud.upload_attachment_to_s3, project.id, instance_ids, background_task_id, db
submission_crud.upload_attachment_to_s3,
project.id,
instance_ids,
background_task_id,
db,
)

if task_id:
Expand Down
8 changes: 3 additions & 5 deletions src/frontend/src/components/createnewproject/DataExtract.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -226,13 +226,11 @@ const DataExtract = ({ flag, customDataExtractUpload, setCustomDataExtractUpload
<div className="fmtm-bg-white lg:fmtm-w-[20%] xl:fmtm-w-[17%] fmtm-px-5 fmtm-py-6">
<h6 className="fmtm-text-xl fmtm-font-[600] fmtm-pb-2 lg:fmtm-pb-6">Map Features</h6>
<p className="fmtm-text-gray-500 lg:fmtm-flex lg:fmtm-flex-col lg:fmtm-gap-3">
<span>
You may either choose to use OSM data, or upload your own data for the mapping project.
</span>
<span>You may either choose to use OSM data, or upload your own data for the mapping project.</span>
<span> The relevant map features that exist on OSM are imported based on the select map area.</span>{' '}
<span>
You can use these map features to use the 'select from map' functionality from ODK that allows you
to select the feature to collect data for.
You can use these map features to use the 'select from map' functionality from ODK that allows you to select
the feature to collect data for.
</span>{' '}
</p>
</div>
Expand Down

0 comments on commit c284271

Please sign in to comment.