From 75ad914ecdf665c6cc64daa89879b00ba4b6e91a Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 11 Sep 2023 17:35:38 +0100 Subject: [PATCH 1/5] ci: move pytest and jest workflows to actions --- .../{workflows => actions}/frontend_test.yml | 9 ++----- .github/{workflows => actions}/pytest.yml | 14 +---------- .github/workflows/pr_frontend_tests.yml | 24 +++++++++++++++++++ .github/workflows/pr_pytest.yml | 24 +++++++++++++++++++ 4 files changed, 51 insertions(+), 20 deletions(-) rename .github/{workflows => actions}/frontend_test.yml (63%) rename .github/{workflows => actions}/pytest.yml (88%) create mode 100644 .github/workflows/pr_frontend_tests.yml create mode 100644 .github/workflows/pr_pytest.yml diff --git a/.github/workflows/frontend_test.yml b/.github/actions/frontend_test.yml similarity index 63% rename from .github/workflows/frontend_test.yml rename to .github/actions/frontend_test.yml index ed407a6ed5..376f851928 100644 --- a/.github/workflows/frontend_test.yml +++ b/.github/actions/frontend_test.yml @@ -1,11 +1,6 @@ name: Frontend Tests -on: - push: - paths: - - "src/frontend/**" # Trigger the workflow only when files within srv/frontend change - - ".github/workflows/**" # Also trigger if github workflow changes - workflow_dispatch: +on: [workflow_call] jobs: test: @@ -14,7 +9,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Node.js uses: actions/setup-node@v2 diff --git a/.github/workflows/pytest.yml b/.github/actions/pytest.yml similarity index 88% rename from .github/workflows/pytest.yml rename to .github/actions/pytest.yml index ed2aa6e829..d600f7d8be 100644 --- a/.github/workflows/pytest.yml +++ b/.github/actions/pytest.yml @@ -1,18 +1,6 @@ name: pytest -on: - # Run tests on all pushed branches - push: - branches: - - "*" - # Run tests on PR, prior to merge to main & development. - pull_request: - branches: - - main - - staging - - development - # Allow manual trigger (workflow_dispatch) - workflow_dispatch: +on: [workflow_call] permissions: contents: read diff --git a/.github/workflows/pr_frontend_tests.yml b/.github/workflows/pr_frontend_tests.yml new file mode 100644 index 0000000000..adc7ad2d95 --- /dev/null +++ b/.github/workflows/pr_frontend_tests.yml @@ -0,0 +1,24 @@ +name: PR + +on: + pull_request: + branches: + - main + - staging + - development + paths: + - src/frontend/** + # Allow manual trigger (workflow_dispatch) + workflow_dispatch: + +jobs: + tests: + runs-on: ubuntu-latest + environment: + name: ${{ github.ref_name }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: PyTest + uses: ./.github/actions/frontend_test.yml diff --git a/.github/workflows/pr_pytest.yml b/.github/workflows/pr_pytest.yml new file mode 100644 index 0000000000..58debc8873 --- /dev/null +++ b/.github/workflows/pr_pytest.yml @@ -0,0 +1,24 @@ +name: PR + +on: + pull_request: + branches: + - main + - staging + - development + paths: + - src/backend/** + # Allow manual trigger (workflow_dispatch) + workflow_dispatch: + +jobs: + tests: + runs-on: ubuntu-latest + environment: + name: ${{ github.ref_name }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: PyTest + uses: ./.github/actions/pytest.yml From 5871741110e0b7166576285bc0e0caf13b38b508 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 11 Sep 2023 17:52:51 +0100 Subject: [PATCH 2/5] ci: reusable test actions --> workflows --- .github/workflows/{pr_frontend_tests.yml => pr.yml} | 9 +++++---- .github/workflows/pr_pytest.yml | 4 ++-- .../reusable/frontend_tests.yml} | 5 ++++- .github/{actions => workflows/reusable}/pytest.yml | 5 ++++- 4 files changed, 15 insertions(+), 8 deletions(-) rename .github/workflows/{pr_frontend_tests.yml => pr.yml} (69%) rename .github/{actions/frontend_test.yml => workflows/reusable/frontend_tests.yml} (88%) rename .github/{actions => workflows/reusable}/pytest.yml (97%) diff --git a/.github/workflows/pr_frontend_tests.yml b/.github/workflows/pr.yml similarity index 69% rename from .github/workflows/pr_frontend_tests.yml rename to .github/workflows/pr.yml index adc7ad2d95..58f124eac0 100644 --- a/.github/workflows/pr_frontend_tests.yml +++ b/.github/workflows/pr.yml @@ -6,13 +6,11 @@ on: - main - staging - development - paths: - - src/frontend/** # Allow manual trigger (workflow_dispatch) workflow_dispatch: jobs: - tests: + pytest: runs-on: ubuntu-latest environment: name: ${{ github.ref_name }} @@ -21,4 +19,7 @@ jobs: uses: actions/checkout@v3 - name: PyTest - uses: ./.github/actions/frontend_test.yml + uses: ./.github/workflows/reusable/pytest.yml + + - name: Frontend Tests + uses: ./.github/workflows/reusable/frontend_test.yml diff --git a/.github/workflows/pr_pytest.yml b/.github/workflows/pr_pytest.yml index 58debc8873..7f1009b55f 100644 --- a/.github/workflows/pr_pytest.yml +++ b/.github/workflows/pr_pytest.yml @@ -12,7 +12,7 @@ on: workflow_dispatch: jobs: - tests: + frontend-tests: runs-on: ubuntu-latest environment: name: ${{ github.ref_name }} @@ -21,4 +21,4 @@ jobs: uses: actions/checkout@v3 - name: PyTest - uses: ./.github/actions/pytest.yml + uses: ./.github/workflows/reusable/pytest.yml diff --git a/.github/actions/frontend_test.yml b/.github/workflows/reusable/frontend_tests.yml similarity index 88% rename from .github/actions/frontend_test.yml rename to .github/workflows/reusable/frontend_tests.yml index 376f851928..52cbdae34d 100644 --- a/.github/actions/frontend_test.yml +++ b/.github/workflows/reusable/frontend_tests.yml @@ -1,6 +1,9 @@ name: Frontend Tests -on: [workflow_call] +on: + workflow_call: + paths: + - src/frontend/** jobs: test: diff --git a/.github/actions/pytest.yml b/.github/workflows/reusable/pytest.yml similarity index 97% rename from .github/actions/pytest.yml rename to .github/workflows/reusable/pytest.yml index d600f7d8be..136aef25cf 100644 --- a/.github/actions/pytest.yml +++ b/.github/workflows/reusable/pytest.yml @@ -1,6 +1,9 @@ name: pytest -on: [workflow_call] +on: + workflow_call: + paths: + - src/backend/** permissions: contents: read From 9c6229cdd9f4030284df873c1b28eb6b78047f29 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 11 Sep 2023 18:08:07 +0100 Subject: [PATCH 3/5] ci: reusable action --> workflow (pytest, jest) --- .github/workflows/pr.yml | 15 +++------------ .github/workflows/reusable/frontend_tests.yml | 3 +-- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 58f124eac0..1e3219854a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -11,15 +11,6 @@ on: jobs: pytest: - runs-on: ubuntu-latest - environment: - name: ${{ github.ref_name }} - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - - name: PyTest - uses: ./.github/workflows/reusable/pytest.yml - - - name: Frontend Tests - uses: ./.github/workflows/reusable/frontend_test.yml + uses: ./.github/workflows/reusable/pytest.yml + frontend-tests: + uses: ./.github/workflows/reusable/frontend_tests.yml diff --git a/.github/workflows/reusable/frontend_tests.yml b/.github/workflows/reusable/frontend_tests.yml index 52cbdae34d..56dd3dab30 100644 --- a/.github/workflows/reusable/frontend_tests.yml +++ b/.github/workflows/reusable/frontend_tests.yml @@ -11,8 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - name: Setup Node.js uses: actions/setup-node@v2 From fa0406a34a4a5cc3189f1fa6147acb07f0cee963 Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 11 Sep 2023 15:51:02 +0100 Subject: [PATCH 4/5] fix: set underpass api url via env var --- src/backend/app/config.py | 1 + src/backend/app/projects/project_crud.py | 6 +++--- src/backend/app/tasks/tasks_crud.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/backend/app/config.py b/src/backend/app/config.py index 56daa0f82a..daa77dc272 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -110,6 +110,7 @@ def assemble_db_connection(cls, v: Optional[str], info: FieldValidationInfo) -> OSM_SCOPE: str = "read_prefs" OSM_LOGIN_REDIRECT_URI: str = "http://127.0.0.1:8080/osmauth/" + UNDERPASS_API_URL: str = "https://raw-data-api0.hotosm.org/v1" SENTRY_DSN: Optional[str] = None model_config = SettingsConfigDict( diff --git a/src/backend/app/projects/project_crud.py b/src/backend/app/projects/project_crud.py index 3aa7a0db13..adc69cdc9b 100644 --- a/src/backend/app/projects/project_crud.py +++ b/src/backend/app/projects/project_crud.py @@ -581,7 +581,7 @@ def get_osm_extracts(boundary: str): else: query["geometry"] = json_boundary - base_url = "https://raw-data-api0.hotosm.org/v1" + base_url = settings.UNDERPASS_API_URL query_url = f"{base_url}/snapshot/" headers = {"accept": "application/json", "Content-Type": "application/json"} @@ -1394,7 +1394,7 @@ def generate_appuser_files( project_log.info(f"Extracting Data from OSM") # OSM Extracts for whole project - pg = PostgresClient("https://raw-data-api0.hotosm.org/v1", "underpass") + pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass") # This file will store osm extracts outfile = f"/tmp/{prefix}_{xform_title}.geojson" @@ -2019,7 +2019,7 @@ async def update_project_form( db.commit() # OSM Extracts for whole project - pg = PostgresClient("https://raw-data-api0.hotosm.org/v1", "underpass") + pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass") outfile = ( f"/tmp/{project_title}_{category}.geojson" # This file will store osm extracts ) diff --git a/src/backend/app/tasks/tasks_crud.py b/src/backend/app/tasks/tasks_crud.py index 21a9d48afb..5aaea42be7 100644 --- a/src/backend/app/tasks/tasks_crud.py +++ b/src/backend/app/tasks/tasks_crud.py @@ -40,6 +40,7 @@ from ..projects import project_crud from ..tasks import tasks_schemas from ..users import user_crud +from app.config import settings async def get_task_count_in_project(db: Session, project_id: int): @@ -330,7 +331,7 @@ async def update_task_files( task_polygons = f"/tmp/{project_name}_{category}_{task_id}.geojson" # Update data extracts in the odk central - pg = PostgresClient("https://raw-data-api0.hotosm.org/v1", "underpass") + pg = PostgresClient(settings.UNDERPASS_API_URL, "underpass") category = "buildings" From 8f812f53c78ca5d46447f71188901c1c36d3c68d Mon Sep 17 00:00:00 2001 From: spwoodcock Date: Mon, 11 Sep 2023 15:51:17 +0100 Subject: [PATCH 5/5] docs: add UNDERPASS_API_URL to env example --- .env.example | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env.example b/.env.example index 3d56c58538..540fe6af62 100644 --- a/.env.example +++ b/.env.example @@ -34,3 +34,6 @@ FMTM_DB_HOST=fmtm-db FMTM_DB_USER=fmtm FMTM_DB_PASSWORD=fmtm FMTM_DB_NAME=fmtm + +### Underpass (optional) ### +# UNDERPASS_API_URL=