From e45e05e5c99898e901249c6b7fd9899ba584ecc4 Mon Sep 17 00:00:00 2001 From: Sam <78538841+spwoodcock@users.noreply.github.com> Date: Sun, 22 Oct 2023 16:15:12 +0100 Subject: [PATCH] ci: update build workflows to use image caching, multi-arch, auto-tag (#925) * refactor: move scripts & josm to contrib dir * ci: add contrib label to labeler.yml * ci: separate frontend and backend tests during pr * ci: remove extract-vars (use auto-tagging) * build: update dockerfiles and compose to use COMMIT_REFs * ci: build backend images on tag and release events * ci: publish docs on changes to development branch * build: set default S3_ENDPOINT for entrypoint (if not set) * build: add healthchecks to dockerfiles * ci: enable backend smoketest using compose --wait * build: update odkcentral --> v2023.4.0 * build: healthcheck for minio service * ci(pytest): update pytest workflow to use img artifacts * build: update odkcentral img, add entrypoint + healthcheck * ci: fix pytest img tar downloading * ci(pytest): fix cached image loading from /tmp * ci(pytest): simplify image caching and loading * ci(pytest): remove image cache upload * ci(pytest): add image cache job to end of workflow * ci(pytest): set cache loading based on needs outputs * ci(pytest): cache images if download was unsuccessful * ci(pytest): fix operator != * build: use env vars for all image versions * build: ODK_CENTRAL_VERSION --> ODK_CENTRAL_TAG * ci(pytest): set image tags for caching from github vars * ci: replace deploy workflow with reusable * ci(pytest): image caching cache_name --> artifact_name * ci(deploy): set .env from var and secrets context (not manual) * ci(pytest): increase timeout for central wait-for-it 30s * ci(pytest): fix timeout syntax * build: add shebang to odkcentral entrypoint script * ci: update env to .env for pytest * ci(pytest): remove additional -alpine from postgis img * build: redact password in odk-central entrypoint * ci: fix to_envs single .env file parsing * ci: pass env var directly to jq, not github var * ci(pytest): fix by using scalar over literal syntax * build: hardcode WORKER_COUNT=1 for odk-central start * ci: update docs workflow to use new cahcing * docs: update info for testing workflows via act * ci(pytest): add caching img steps directly to workflow (no reuse) * ci(pytest): typo image --> images for cache key * refactor: pydantic deprecation FIeldValidationInfo --> ValidationInfo * ci: rewrite pytest to build pr image prior to test * ci: don't push,cache,multi_arch PR image builds * ci: add permissions: write to pytest build + scan=false * ci: remove permissions from pytest (handle downstream) * ci: update conditionals for building pr images * ci: update pytest cache-hit output * ci: get image_tag instead of image_name from output * ci(pytest): fix case where cache hit, skip build * ci(pytest): set BACKEND_IMG_TAG in all if cases * ci(pytest): replace BACKEND_IMG_TAG --> API_TAG_OVERRIDE * ci(pytest): fix getting backend image tag from docker img ls * ci(pytest): only run image caching steps if explicity cache=false * ci(pytest): revert ==false to !=true negation * ci: replace all refs to repo with gh vars * ci(pytest): prep pytest for moving to gh-workflows * ci(pytest): test with caching off * ci(pytest): remove redundant var build_test_img * ci(pytest): fix "" workflow syntax --> '' * ci(pytest): quote ci-development fallback * ci(pytest): move where default ci-development tag is set * ci(pytest): reenable caching for workflow images * ci(pytest): inputs.cache_imgs to array * ci(pytest): allow passing of image_tag to image build stage * ci: replace all reusable workflows with hotosm/gh-workflows * ci(pytest): use new format of reusable pytest workflow * build: rename API_TAG_OVERRIDE --> TAG_OVERRIDE --- .env.example | 1 - .github/labeler.yml | 3 +- .github/workflows/build_and_deploy.yml | 131 +++++------------- .github/workflows/build_ci_img.yml | 10 +- .github/workflows/build_odk_imgs.yml | 10 +- .github/workflows/docs.yml | 22 +-- .github/workflows/pr_test.yml | 23 --- .github/workflows/pr_test_backend.yml | 31 +++++ .github/workflows/pr_test_frontend.yml | 19 +++ .github/workflows/r-extract_vars.yml | 53 ------- .github/workflows/r-frontend_tests.yml | 32 ----- .github/workflows/r-pytest.yml | 102 -------------- .github/workflows/tag_build.yml | 25 ++++ .github/workflows/tests/push_payload.json | 3 +- .github/workflows/tests/test_ci.sh | 9 +- .github/workflows/wiki.yml | 2 +- contrib/README.md | 4 + {josm => contrib/josm}/Dockerfile | 0 contrib/josm/README.md | 12 ++ .../josm}/container-entrypoint.sh | 0 {josm => contrib/josm}/docker-compose.yml | 0 {josm => contrib/josm}/nginx-josm.conf | 0 {josm => contrib/josm}/novnc/Dockerfile | 0 {josm => contrib/josm}/novnc/index.html | 0 {josm => contrib/josm}/preferences.xml | 0 .../scripts/odk_fieldmap_original/README.md | 5 + .../odk_fieldmap_original/parse_pip_error.py | 0 .../scripts}/odk_fieldmap_original/setup.sh | 0 .../odk_fieldmap_original/utils/aoi2odk.py | 0 .../utils/dl_submissions.py | 0 .../utils/example_odk_requests.py | 0 .../utils/forms2server.py | 0 .../odk_fieldmap_original/utils/geo_utils.py | 0 .../utils/odk_requests.py | 0 .../utils/old_qr_stuff.py | 0 .../odk_fieldmap_original/utils/overpass.py | 0 .../odk_fieldmap_original/utils/renamer.py | 0 .../utils/tasks2forms.py | 0 .../building_clusters_of_5.sql | 0 .../Examples_and_tests/centroids.sql | 0 .../Examples_and_tests/clean_and_simplify.sql | 0 .../Examples_and_tests/clustering.sql | 0 ...ve_hull_tasks_from_clustered_buildings.sql | 0 .../count_building_tags.sql | 0 ...lygons_with_no_features_into_neighbors.sql | 0 .../Examples_and_tests/points_in_polygon.sql | 0 .../Examples_and_tests/polygonize.sql | 0 .../select_features_in_polygon.sql | 0 .../split_aoi_by_osm_lines.sql | 0 .../split_area_by_osm_lines.sql | 0 .../task_splitting_optimized.sql | 0 .../Examples_and_tests/voronoi.sql | 0 contrib/scripts/postgis_snippets/README.md | 3 + .../import_geojson_as_postgis_with_jsonb.md | 0 .../postgis_snippets/postgis_resources.md | 0 ...01_split_AOI_by_existing_line_features.sql | 0 ...it_02_count_buildings_for_subsplitting.sql | 0 .../fmtm-split_03_cluster_buildings.sql | 0 ...te_polygons_around_clustered_buildings.sql | 0 .../task_splitting_for_osm_buildings.sql | 0 .../task_splitting_for_osm_roads.sql | 0 .../task_splitting/task_splitting_readme.md | 0 docker-compose.development.yml | 21 +-- docker-compose.main.yml | 13 +- docker-compose.noodk.yml | 14 +- docker-compose.yml | 30 ++-- gen-env.sh | 2 - odkcentral/api/Dockerfile | 52 +++---- odkcentral/api/init-user-and-start.sh | 25 ++-- odkcentral/proxy/Dockerfile | 3 + src/backend/Dockerfile | 5 + src/backend/app-entrypoint.sh | 2 +- src/backend/app/config.py | 6 +- src/frontend/prod.dockerfile | 2 + 74 files changed, 253 insertions(+), 422 deletions(-) delete mode 100644 .github/workflows/pr_test.yml create mode 100644 .github/workflows/pr_test_backend.yml create mode 100644 .github/workflows/pr_test_frontend.yml delete mode 100644 .github/workflows/r-extract_vars.yml delete mode 100644 .github/workflows/r-frontend_tests.yml delete mode 100644 .github/workflows/r-pytest.yml create mode 100644 .github/workflows/tag_build.yml create mode 100644 contrib/README.md rename {josm => contrib/josm}/Dockerfile (100%) create mode 100644 contrib/josm/README.md rename {josm => contrib/josm}/container-entrypoint.sh (100%) rename {josm => contrib/josm}/docker-compose.yml (100%) rename {josm => contrib/josm}/nginx-josm.conf (100%) rename {josm => contrib/josm}/novnc/Dockerfile (100%) rename {josm => contrib/josm}/novnc/index.html (100%) rename {josm => contrib/josm}/preferences.xml (100%) create mode 100644 contrib/scripts/odk_fieldmap_original/README.md rename {scripts => contrib/scripts}/odk_fieldmap_original/parse_pip_error.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/setup.sh (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/aoi2odk.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/dl_submissions.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/example_odk_requests.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/forms2server.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/geo_utils.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/odk_requests.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/old_qr_stuff.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/overpass.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/renamer.py (100%) rename {scripts => contrib/scripts}/odk_fieldmap_original/utils/tasks2forms.py (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/centroids.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/clean_and_simplify.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/clustering.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/count_building_tags.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/points_in_polygon.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/polygonize.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/Examples_and_tests/voronoi.sql (100%) create mode 100644 contrib/scripts/postgis_snippets/README.md rename {scripts => contrib/scripts}/postgis_snippets/import_geojson_as_postgis_with_jsonb.md (100%) rename {scripts => contrib/scripts}/postgis_snippets/postgis_resources.md (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql (100%) rename {scripts => contrib/scripts}/postgis_snippets/task_splitting/task_splitting_readme.md (100%) diff --git a/.env.example b/.env.example index e7a20bbd2b..1159a419c4 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,6 @@ ### copy to .env and set variables ### ODK Central ### -ODK_CENTRAL_VERSION=v2023.2.1 ODK_CENTRAL_URL=https://central-proxy ODK_CENTRAL_USER=dev@fmtm.hotosm.org ODK_CENTRAL_PASSWD=testuserpassword diff --git a/.github/labeler.yml b/.github/labeler.yml index 90d3afaeeb..3407c15cf5 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -9,7 +9,6 @@ - "**/Dockerfile" - "**/*.dockerfile" - "**/*entrypoint.sh" - - "josm/**/*" "migration": - "src/backend/migrations/**/*" "documentation": @@ -20,3 +19,5 @@ - "INSTALL.md" "ODK": - "odkcentral/**/*" +"contrib": + - "contrib/**/*" diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 3e1f0a2247..fb11d8897f 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -1,3 +1,5 @@ +# Workflow for build and auto-deploy of branches + name: Build and Deploy on: @@ -15,52 +17,47 @@ on: jobs: pytest: - uses: ./.github/workflows/r-pytest.yml + uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main with: - image_tag: ci-${{ github.ref_name }} + image_name: ghcr.io/${{ github.repository }}/backend + build_context: src/backend + build_args: | + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} + docker_compose_service: api + tag_override: ci-${{ github.ref_name }} secrets: inherit frontend-tests: - uses: ./.github/workflows/r-frontend_tests.yml - - extract-vars: - needs: - - pytest - - frontend-tests - uses: ./.github/workflows/r-extract_vars.yml + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main with: - environment: ${{ github.ref_name }} + working_dir: src/frontend backend-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - needs: [extract-vars] with: context: src/backend build_target: prod - image_tags: | - "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-${{ github.ref_name }}" - "ghcr.io/hotosm/fmtm/backend:latest" + image_name: ghcr.io/${{ github.repository }}/backend build_args: | - APP_VERSION=${{ needs.extract-vars.outputs.api_version }} + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} frontend-main-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - needs: [extract-vars] with: context: src/frontend dockerfile: prod.dockerfile build_target: prod - image_tags: | - "ghcr.io/hotosm/fmtm/frontend:${{ needs.extract-vars.outputs.frontend_main_version }}-${{ github.ref_name }}" - "ghcr.io/hotosm/fmtm/frontend:latest" + image_name: ghcr.io/${{ github.repository }}/frontend build_args: | - APP_VERSION=${{ needs.extract-vars.outputs.frontend_main_version }} - VITE_API_URL=${{ needs.extract-vars.outputs.api_url }} + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} + VITE_API_URL=${{ vars.URL_SCHEME }}://${{ vars.API_URL }}" smoke-test-backend: runs-on: ubuntu-latest needs: - - extract-vars - backend-build environment: name: ${{ github.ref_name }} @@ -69,55 +66,28 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Environment to .env + - name: Vars and Secrets to Env env: GIT_BRANCH: ${{ github.ref_name }} - API_VERSION: ${{ needs.extract-vars.outputs.api_version }} - FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }} + VARS_CONTEXT: ${{ toJson(vars) }} + SECRETS_CONTEXT: ${{ toJson(secrets) }} run: | - echo "${{ secrets.DOTENV }}" > .env - echo "GIT_BRANCH=${GIT_BRANCH}" >> .env - echo "API_VERSION=${API_VERSION}" >> .env - echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env + to_envs() { jq -r "to_entries[] | \"\(.key)=\(.value)\""; } + + echo "GIT_BRANCH=${GIT_BRANCH}" >> $GITHUB_ENV + echo "${VARS_CONTEXT}" | to_envs >> $GITHUB_ENV + echo "${SECRETS_CONTEXT}" | to_envs >> $GITHUB_ENV + + - name: Create .env file + run: env > .env - name: Backend smoke test run: | - echo "Not implemented" - # source .env - # docker network create fmtm - - # docker pull "postgis/postgis:14-3.3-alpine" - # docker run --rm -d \ - # --name=fmtm-db \ - # --network=fmtm \ - # -e POSTGRES_USER=fmtm \ - # -e POSTGRES_PASSWORD=fmtm \ - # -e POSTGRES_DB=fmtm \ - # "postgis/postgis:14-3.3-alpine" - - # docker pull "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" - # docker run --rm -d \ - # --network=fmtm \ - # -p 8080:8080 \ - # -e OSM_CLIENT_ID="test" \ - # -e OSM_CLIENT_SECRET="test" \ - # -e OSM_SECRET_KEY="test" \ - # -e S3_ACCESS_KEY="fmtm" \ - # -e S3_SECRET_KEY="somelongpassword" \ - # "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" - - # # First wait 10 seconds for API - # sleep 10 - # # Check the exit status of curl and exit the job if it fails - # if ! curl -f http://localhost:8080/docs; then - # echo "curl failed to access http://localhost:8080/docs" - # exit 1 - # fi + docker compose up -d api --wait --wait-timeout 60 smoke-test-frontend: runs-on: ubuntu-latest needs: - - extract-vars - frontend-main-build environment: name: ${{ github.ref_name }} @@ -130,40 +100,11 @@ jobs: run: echo "Not implemented" deploy-containers: - runs-on: ubuntu-latest needs: - - extract-vars - smoke-test-backend - smoke-test-frontend - environment: - name: ${{ github.ref_name }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Environment to .env - env: - GIT_BRANCH: ${{ github.ref_name }} - API_VERSION: ${{ needs.extract-vars.outputs.api_version }} - FRONTEND_MAIN_VERSION: ${{ needs.extract-vars.outputs.frontend_main_version }} - run: | - echo "${{ secrets.DOTENV }}" > .env - echo "GIT_BRANCH=${GIT_BRANCH}" >> .env - echo "API_VERSION=${API_VERSION}" >> .env - echo "FRONTEND_MAIN_VERSION=${FRONTEND_MAIN_VERSION}" >> .env - - - uses: webfactory/ssh-agent@v0.7.0 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Disable Host key verification - # Hack to prevent "Host key verification failed". Should be replaced with a ssh-keyscan based solution - run: echo "StrictHostKeyChecking no" >> ~/.ssh/config - - - name: Deploy - run: | - docker compose --file "docker-compose.${{ github.ref_name }}.yml" pull - docker compose --file "docker-compose.${{ github.ref_name }}.yml" up --detach --remove-orphans - env: - DOCKER_HOST: ${{ vars.DOCKER_HOST }} + uses: hotosm/gh-workflows/.github/workflows/remote_deploy.yml@main + with: + environment: ${{ github.ref_name }} + docker_compose_file: "docker-compose.${{ github.ref_name }}.yml" + secrets: inherit diff --git a/.github/workflows/build_ci_img.yml b/.github/workflows/build_ci_img.yml index cae03d55a9..385ccce5e2 100644 --- a/.github/workflows/build_ci_img.yml +++ b/.github/workflows/build_ci_img.yml @@ -15,17 +15,13 @@ on: workflow_dispatch: jobs: - extract-vars: - uses: ./.github/workflows/r-extract_vars.yml - backend-ci-build: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main - needs: [extract-vars] with: context: src/backend build_target: ci image_tags: | - "ghcr.io/hotosm/fmtm/backend:${{ needs.extract-vars.outputs.api_version }}-ci-${{ github.ref_name }}" - "ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }}" + "ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }}" build_args: | - APP_VERSION=${{ needs.extract-vars.outputs.api_version }} + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} diff --git a/.github/workflows/build_odk_imgs.yml b/.github/workflows/build_odk_imgs.yml index 14832bfd00..09ca4f6562 100644 --- a/.github/workflows/build_odk_imgs.yml +++ b/.github/workflows/build_odk_imgs.yml @@ -17,15 +17,15 @@ jobs: with: context: odkcentral/api image_tags: | - "ghcr.io/hotosm/fmtm/odkcentral:${{ vars.ODK_CENTRAL_VERSION }}" - "ghcr.io/hotosm/fmtm/odkcentral:latest" + "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" + "ghcr.io/${{ github.repository }}/odkcentral:latest" build_args: | - ODK_CENTRAL_VERSION=${{ vars.ODK_CENTRAL_VERSION }} + ODK_CENTRAL_TAG=${{ vars.ODK_CENTRAL_TAG }} build-proxy: uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main with: context: odkcentral/proxy image_tags: | - "ghcr.io/hotosm/fmtm/odkcentral-proxy:${{ vars.ODK_CENTRAL_VERSION }}" - "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest" + "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" + "ghcr.io/${{ github.repository }}/odkcentral-proxy:latest" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1b5461d8e6..34ec15e157 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -6,7 +6,7 @@ on: - docs/** - src/** - mkdocs.yml - branches: [main] + branches: [development] # Allow manual trigger (workflow_dispatch) workflow_dispatch: @@ -14,20 +14,14 @@ jobs: build_doxygen: uses: hotosm/gh-workflows/.github/workflows/doxygen_build.yml@main with: - cache_paths: | - docs/apidocs - docs/openapi.json - cache_key: docs-build + output_path: docs/apidocs build_openapi_json: uses: hotosm/gh-workflows/.github/workflows/openapi_build.yml@main with: - image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }} + image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }} example_env_file_path: ".env.example" - cache_paths: | - docs/apidocs - docs/openapi.json - cache_key: docs-build + output_path: docs/openapi.json publish_docs: uses: hotosm/gh-workflows/.github/workflows/mkdocs_build.yml@main @@ -35,8 +29,6 @@ jobs: - build_doxygen - build_openapi_json with: - image: ghcr.io/hotosm/fmtm/backend:ci-${{ github.ref_name }} - cache_paths: | - docs/apidocs - docs/openapi.json - cache_key: docs-build + image: ghcr.io/${{ github.repository }}/backend:ci-${{ github.ref_name }} + doxygen: true + openapi: true diff --git a/.github/workflows/pr_test.yml b/.github/workflows/pr_test.yml deleted file mode 100644 index 420287420c..0000000000 --- a/.github/workflows/pr_test.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: PR Tests - -on: - pull_request: - branches: - - main - - staging - - development - # Workflow is triggered only if src changes - paths: - - src/** - # Allow manual trigger (workflow_dispatch) - workflow_dispatch: - -jobs: - pytest: - uses: ./.github/workflows/r-pytest.yml - with: - image_tag: ci-${{ github.base_ref }} - secrets: inherit - - frontend-tests: - uses: ./.github/workflows/r-frontend_tests.yml diff --git a/.github/workflows/pr_test_backend.yml b/.github/workflows/pr_test_backend.yml new file mode 100644 index 0000000000..9b9591c006 --- /dev/null +++ b/.github/workflows/pr_test_backend.yml @@ -0,0 +1,31 @@ +name: PR Test Backend + +on: + pull_request: + branches: + - main + - staging + - development + # Workflow is triggered only if src/backend changes + paths: + - src/backend/** + # Allow manual trigger (workflow_dispatch) + workflow_dispatch: + +jobs: + pytest: + uses: hotosm/gh-workflows/.github/workflows/test_pytest_compose.yml@main + with: + image_name: ghcr.io/${{ github.repository }}/backend + build_context: src/backend + build_args: | + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} + docker_compose_service: api + cache_extra_imgs: | + "docker.io/postgis/postgis:${{ vars.POSTGIS_TAG }}" + "docker.io/minio/minio:${{ vars.MINIO_TAG }}" + # For caching odk central images, add: + # "ghcr.io/${{ github.repository }}/odkcentral:${{ vars.ODK_CENTRAL_TAG }}" + # "ghcr.io/${{ github.repository }}/odkcentral-proxy:${{ vars.ODK_CENTRAL_TAG }}" + secrets: inherit diff --git a/.github/workflows/pr_test_frontend.yml b/.github/workflows/pr_test_frontend.yml new file mode 100644 index 0000000000..0a2bb7bcdb --- /dev/null +++ b/.github/workflows/pr_test_frontend.yml @@ -0,0 +1,19 @@ +name: PR Tests Frontend + +on: + pull_request: + branches: + - main + - staging + - development + # Workflow is triggered only if src/frontend changes + paths: + - src/frontend/** + # Allow manual trigger (workflow_dispatch) + workflow_dispatch: + +jobs: + frontend-tests: + uses: hotosm/gh-workflows/.github/workflows/test_pnpm.yml@main + with: + working_dir: src/frontend diff --git a/.github/workflows/r-extract_vars.yml b/.github/workflows/r-extract_vars.yml deleted file mode 100644 index e4151ae9fd..0000000000 --- a/.github/workflows/r-extract_vars.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: Extract Project Variables - -on: - workflow_call: - inputs: - environment: - description: "The GitHub environment to extract vars from." - required: false - type: string - default: "" - outputs: - api_version: - description: "Backend API Version." - value: ${{ jobs.extract-vars.outputs.api_version }} - frontend_main_version: - description: "Frontend Version." - value: ${{ jobs.extract-vars.outputs.frontend_main_version }} - api_url: - description: "URL to access the backend API." - value: ${{ jobs.extract-vars.outputs.api_url }} - -jobs: - extract-vars: - runs-on: ubuntu-latest - environment: ${{ inputs.environment }} - outputs: - api_version: ${{ steps.extract_api_version.outputs.api_version }} - frontend_main_version: ${{ steps.extract_frontend_version.outputs.frontend_main_version }} - api_url: ${{ steps.get_env_vars.outputs.api_url }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Extract api version - id: extract_api_version - working-directory: src/backend - run: | - API_VERSION=$(python -c 'from app.__version__ import __version__; print(__version__)') - echo "api_version=${API_VERSION}" >> $GITHUB_OUTPUT - - - name: Extract frontend versions - id: extract_frontend_version - working-directory: src/frontend - run: | - FRONTEND_MAIN_VERSION=$(jq -r '.version' package.json) - echo "frontend_main_version=${FRONTEND_MAIN_VERSION}" >> $GITHUB_OUTPUT - - - name: Get environment vars - id: get_env_vars - run: | - echo "api_url: ${{ vars.URL_SCHEME }}://${{ vars.API_URL }}" - echo "api_url=${{ vars.URL_SCHEME }}://${{ vars.API_URL }}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/r-frontend_tests.yml b/.github/workflows/r-frontend_tests.yml deleted file mode 100644 index 0d62d57a0e..0000000000 --- a/.github/workflows/r-frontend_tests.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Frontend Tests - -on: - workflow_call: - -jobs: - test: - name: Run Frontend Tests - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - uses: pnpm/action-setup@v2 - with: - version: 8 - - - name: Setup Node.js - uses: actions/setup-node@v3 - with: - node-version: 18 - cache: "pnpm" - cache-dependency-path: "src/frontend/pnpm-lock.yaml" - - - name: Install dependencies - working-directory: src/frontend - run: pnpm install - - - name: Test Frontend - working-directory: src/frontend - run: pnpm run test diff --git a/.github/workflows/r-pytest.yml b/.github/workflows/r-pytest.yml deleted file mode 100644 index bd5bf87b85..0000000000 --- a/.github/workflows/r-pytest.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: pytest - -on: - workflow_call: - inputs: - image_tag: - required: true - type: string - environment: - required: false - type: string - -permissions: - contents: read - -jobs: - cache-img-postgis: - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_name: postgis/postgis:14-3.3-alpine - cache_key: img-postgis - - cache-img-odk: - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_name: ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1 - cache_key: img-odk - - cache-img-odk-proxy: - uses: hotosm/gh-workflows/.github/workflows/image_cache.yml@main - with: - image_name: ghcr.io/hotosm/fmtm/odkcentral-proxy:latest - cache_key: img-odk-proxy - - run-pytest: - runs-on: ubuntu-latest - environment: - name: ${{ inputs.environment || 'test' }} - needs: - - cache-img-postgis - - cache-img-odk - - cache-img-odk-proxy - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Restore Img Caches - id: restore-imgs - uses: actions/cache@v3 - with: - path: | - ${{ needs.cache-img-postgis.outputs.cache_path }} - ${{ needs.cache-img-odk.outputs.cache_path }} - ${{ needs.cache-img-odk-proxy-cache.outputs.cache_path }} - key: img- - restore-keys: | - ${{ needs.cache-img-postgis.outputs.cache_key }} - ${{ needs.cache-img-odk.outputs.cache_key }} - ${{ needs.cache-img-odk-proxy-cache.outputs.cache_key }} - - - name: Load Cached Imgs - if: steps.restore-imgs.outputs.cache-hit == 'true' - run: | - docker image load --input ${{ needs.cache-img-postgis.outputs.cache_path }} || true - docker image load --input ${{ needs.cache-img-odk.outputs.cache_path }} || true - docker image load --input ${{ needs.cache-img-odk-proxy-cache.outputs.cache_path }} || true - - - name: Environment to .env - env: - DEBUG: True - LOG_LEVEL: DEBUG - API_TAG_OVERRIDE: "${{ inputs.image_tag }}" - FRONTEND_MAIN_URL: "${{ vars.FRONTEND_MAIN_URL }}" - ODK_CENTRAL_URL: "${{ vars.ODK_CENTRAL_URL }}" - ODK_CENTRAL_USER: "${{ vars.ODK_CENTRAL_USER }}" - ODK_CENTRAL_PASSWD: "${{ secrets.ODK_CENTRAL_PASSWD }}" - OSM_CLIENT_ID: "${{ secrets.OSM_CLIENT_ID }}" - OSM_CLIENT_SECRET: "${{ secrets.OSM_CLIENT_SECRET }}" - OSM_SECRET_KEY: "${{ secrets.OSM_SECRET_KEY }}" - S3_ACCESS_KEY: "fmtm" - S3_SECRET_KEY: "somelongpassword" - run: | - echo "DEBUG=${DEBUG}" >> .env - echo "LOG_LEVEL=${LOG_LEVEL}" >> .env - echo "API_TAG_OVERRIDE=${API_TAG_OVERRIDE}" >> .env - echo "FRONTEND_MAIN_URL=${FRONTEND_MAIN_URL}" >> .env - echo "ODK_CENTRAL_URL=${ODK_CENTRAL_URL}" >> .env - echo "ODK_CENTRAL_USER=${ODK_CENTRAL_USER}" >> .env - echo "ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}" >> .env - echo "OSM_CLIENT_ID=${OSM_CLIENT_ID}" >> .env - echo "OSM_CLIENT_SECRET=${OSM_CLIENT_SECRET}" >> .env - echo "OSM_SECRET_KEY=${OSM_SECRET_KEY}" >> .env - echo "S3_ACCESS_KEY=${S3_ACCESS_KEY}" >> .env - echo "S3_SECRET_KEY=${S3_SECRET_KEY}" >> .env - - - name: Run PyTest - run: | - docker compose run api \ - wait-for-it fmtm-db:5432 --strict \ - -- wait-for-it central:8383 --strict \ - -- pytest diff --git a/.github/workflows/tag_build.yml b/.github/workflows/tag_build.yml new file mode 100644 index 0000000000..6e5b35a8dd --- /dev/null +++ b/.github/workflows/tag_build.yml @@ -0,0 +1,25 @@ +name: Build Tag & Release Images + +on: + # Any tag push + push: + tags: + - "*" + # All releases + release: + types: [published] + # Allow manual trigger + workflow_dispatch: + +jobs: + backend-build: + uses: hotosm/gh-workflows/.github/workflows/image_build.yml@main + with: + context: src/backend + build_target: prod + image_name: ghcr.io/${{ github.repository }}/backend + build_args: | + APP_VERSION=${{ github.ref_name }} + COMMIT_REF=${{ github.sha }} + +# Frontend is not built as build variables are required diff --git a/.github/workflows/tests/push_payload.json b/.github/workflows/tests/push_payload.json index eb2701ab2c..d476f82346 100644 --- a/.github/workflows/tests/push_payload.json +++ b/.github/workflows/tests/push_payload.json @@ -1,3 +1,4 @@ { - "base_ref ": "development" + "base_ref ": "development", + "ref": "refs/heads/development" } diff --git a/.github/workflows/tests/test_ci.sh b/.github/workflows/tests/test_ci.sh index a743734438..60a319231d 100644 --- a/.github/workflows/tests/test_ci.sh +++ b/.github/workflows/tests/test_ci.sh @@ -11,8 +11,13 @@ set -e # GITHUB_TOKEN=input # Feed to act using -s flag: -s GITHUB_TOKEN=input_personal_access_token -# PR -act pull_request -W .github/workflows/pr_test.yml \ +# PR Test Backend +act pull_request -W .github/workflows/pr_test_backend.yml \ + -e .github/workflows/tests/pr_payload.json \ + --var-file=.env --secret-file=.env + +# PR Test Frontend +act pull_request -W .github/workflows/pr_test_frontend.yml \ -e .github/workflows/tests/pr_payload.json \ --var-file=.env --secret-file=.env diff --git a/.github/workflows/wiki.yml b/.github/workflows/wiki.yml index d52f0a2266..f6674ca498 100644 --- a/.github/workflows/wiki.yml +++ b/.github/workflows/wiki.yml @@ -4,7 +4,7 @@ on: push: paths: - docs/** - branches: [main] + branches: [development] # Allow manual trigger (workflow_dispatch) workflow_dispatch: diff --git a/contrib/README.md b/contrib/README.md new file mode 100644 index 0000000000..bb52ce2cf8 --- /dev/null +++ b/contrib/README.md @@ -0,0 +1,4 @@ +# Contrib + +- Additional content not related to the main functionality of FMTM. +- External contributions not merged into core repo. diff --git a/josm/Dockerfile b/contrib/josm/Dockerfile similarity index 100% rename from josm/Dockerfile rename to contrib/josm/Dockerfile diff --git a/contrib/josm/README.md b/contrib/josm/README.md new file mode 100644 index 0000000000..e5cde779c4 --- /dev/null +++ b/contrib/josm/README.md @@ -0,0 +1,12 @@ +# JOSM + +- Container images for running JOSM in a remote container. + +## Use Cases + +- Control JOSM remotely via it's API. +- Accessing JOSM remotely via a web browser, without having to install it. + +## Usage + +See `docs/dev/Backend.md`. diff --git a/josm/container-entrypoint.sh b/contrib/josm/container-entrypoint.sh similarity index 100% rename from josm/container-entrypoint.sh rename to contrib/josm/container-entrypoint.sh diff --git a/josm/docker-compose.yml b/contrib/josm/docker-compose.yml similarity index 100% rename from josm/docker-compose.yml rename to contrib/josm/docker-compose.yml diff --git a/josm/nginx-josm.conf b/contrib/josm/nginx-josm.conf similarity index 100% rename from josm/nginx-josm.conf rename to contrib/josm/nginx-josm.conf diff --git a/josm/novnc/Dockerfile b/contrib/josm/novnc/Dockerfile similarity index 100% rename from josm/novnc/Dockerfile rename to contrib/josm/novnc/Dockerfile diff --git a/josm/novnc/index.html b/contrib/josm/novnc/index.html similarity index 100% rename from josm/novnc/index.html rename to contrib/josm/novnc/index.html diff --git a/josm/preferences.xml b/contrib/josm/preferences.xml similarity index 100% rename from josm/preferences.xml rename to contrib/josm/preferences.xml diff --git a/contrib/scripts/odk_fieldmap_original/README.md b/contrib/scripts/odk_fieldmap_original/README.md new file mode 100644 index 0000000000..659819b805 --- /dev/null +++ b/contrib/scripts/odk_fieldmap_original/README.md @@ -0,0 +1,5 @@ +# ODK Fieldmap + +Various scripts for interacting with ODK and +FMTM to automate tasks & produce useful outputs for +GIS visualisation. diff --git a/scripts/odk_fieldmap_original/parse_pip_error.py b/contrib/scripts/odk_fieldmap_original/parse_pip_error.py similarity index 100% rename from scripts/odk_fieldmap_original/parse_pip_error.py rename to contrib/scripts/odk_fieldmap_original/parse_pip_error.py diff --git a/scripts/odk_fieldmap_original/setup.sh b/contrib/scripts/odk_fieldmap_original/setup.sh similarity index 100% rename from scripts/odk_fieldmap_original/setup.sh rename to contrib/scripts/odk_fieldmap_original/setup.sh diff --git a/scripts/odk_fieldmap_original/utils/aoi2odk.py b/contrib/scripts/odk_fieldmap_original/utils/aoi2odk.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/aoi2odk.py rename to contrib/scripts/odk_fieldmap_original/utils/aoi2odk.py diff --git a/scripts/odk_fieldmap_original/utils/dl_submissions.py b/contrib/scripts/odk_fieldmap_original/utils/dl_submissions.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/dl_submissions.py rename to contrib/scripts/odk_fieldmap_original/utils/dl_submissions.py diff --git a/scripts/odk_fieldmap_original/utils/example_odk_requests.py b/contrib/scripts/odk_fieldmap_original/utils/example_odk_requests.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/example_odk_requests.py rename to contrib/scripts/odk_fieldmap_original/utils/example_odk_requests.py diff --git a/scripts/odk_fieldmap_original/utils/forms2server.py b/contrib/scripts/odk_fieldmap_original/utils/forms2server.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/forms2server.py rename to contrib/scripts/odk_fieldmap_original/utils/forms2server.py diff --git a/scripts/odk_fieldmap_original/utils/geo_utils.py b/contrib/scripts/odk_fieldmap_original/utils/geo_utils.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/geo_utils.py rename to contrib/scripts/odk_fieldmap_original/utils/geo_utils.py diff --git a/scripts/odk_fieldmap_original/utils/odk_requests.py b/contrib/scripts/odk_fieldmap_original/utils/odk_requests.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/odk_requests.py rename to contrib/scripts/odk_fieldmap_original/utils/odk_requests.py diff --git a/scripts/odk_fieldmap_original/utils/old_qr_stuff.py b/contrib/scripts/odk_fieldmap_original/utils/old_qr_stuff.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/old_qr_stuff.py rename to contrib/scripts/odk_fieldmap_original/utils/old_qr_stuff.py diff --git a/scripts/odk_fieldmap_original/utils/overpass.py b/contrib/scripts/odk_fieldmap_original/utils/overpass.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/overpass.py rename to contrib/scripts/odk_fieldmap_original/utils/overpass.py diff --git a/scripts/odk_fieldmap_original/utils/renamer.py b/contrib/scripts/odk_fieldmap_original/utils/renamer.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/renamer.py rename to contrib/scripts/odk_fieldmap_original/utils/renamer.py diff --git a/scripts/odk_fieldmap_original/utils/tasks2forms.py b/contrib/scripts/odk_fieldmap_original/utils/tasks2forms.py similarity index 100% rename from scripts/odk_fieldmap_original/utils/tasks2forms.py rename to contrib/scripts/odk_fieldmap_original/utils/tasks2forms.py diff --git a/scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/building_clusters_of_5.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/centroids.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/centroids.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/centroids.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/centroids.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/clean_and_simplify.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/clustering.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/clustering.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/clustering.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/clustering.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/concave_hull_tasks_from_clustered_buildings.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/count_building_tags.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/merge_polygons_with_no_features_into_neighbors.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/points_in_polygon.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/polygonize.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/polygonize.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/polygonize.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/polygonize.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/select_features_in_polygon.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/split_aoi_by_osm_lines.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/split_area_by_osm_lines.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/task_splitting_optimized.sql diff --git a/scripts/postgis_snippets/Examples_and_tests/voronoi.sql b/contrib/scripts/postgis_snippets/Examples_and_tests/voronoi.sql similarity index 100% rename from scripts/postgis_snippets/Examples_and_tests/voronoi.sql rename to contrib/scripts/postgis_snippets/Examples_and_tests/voronoi.sql diff --git a/contrib/scripts/postgis_snippets/README.md b/contrib/scripts/postgis_snippets/README.md new file mode 100644 index 0000000000..bd0339bf3f --- /dev/null +++ b/contrib/scripts/postgis_snippets/README.md @@ -0,0 +1,3 @@ +# PostGIS Snippets + +Various useful PostGIS SQL commands contributed by @ivangayton. diff --git a/scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md b/contrib/scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md similarity index 100% rename from scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md rename to contrib/scripts/postgis_snippets/import_geojson_as_postgis_with_jsonb.md diff --git a/scripts/postgis_snippets/postgis_resources.md b/contrib/scripts/postgis_snippets/postgis_resources.md similarity index 100% rename from scripts/postgis_snippets/postgis_resources.md rename to contrib/scripts/postgis_snippets/postgis_resources.md diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_01_split_AOI_by_existing_line_features.sql diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_02_count_buildings_for_subsplitting.sql diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_03_cluster_buildings.sql diff --git a/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql b/contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql rename to contrib/scripts/postgis_snippets/task_splitting/fmtm_task_splitting_for_buildings/fmtm-split_04_create_polygons_around_clustered_buildings.sql diff --git a/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql b/contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql rename to contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_buildings.sql diff --git a/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql b/contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql similarity index 100% rename from scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql rename to contrib/scripts/postgis_snippets/task_splitting/task_splitting_for_osm_roads.sql diff --git a/scripts/postgis_snippets/task_splitting/task_splitting_readme.md b/contrib/scripts/postgis_snippets/task_splitting/task_splitting_readme.md similarity index 100% rename from scripts/postgis_snippets/task_splitting/task_splitting_readme.md rename to contrib/scripts/postgis_snippets/task_splitting/task_splitting_readme.md diff --git a/docker-compose.development.yml b/docker-compose.development.yml index a05cfcf5c4..1e5710611b 100644 --- a/docker-compose.development.yml +++ b/docker-compose.development.yml @@ -64,7 +64,7 @@ services: - "traefik.http.routers.http_catchall.middlewares=https_redirect" fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ @@ -79,12 +79,7 @@ services: restart: "unless-stopped" api: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" - build: - context: src/backend - target: prod - args: - APP_VERSION: ${API_VERSION} + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_api volumes: - fmtm_logs:/opt/logs @@ -108,7 +103,7 @@ services: - "traefik.http.routers.api.service=api-svc" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_migrations depends_on: - fmtm-db @@ -120,7 +115,7 @@ services: restart: "on-failure:3" ui: - image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH}" build: context: src/frontend dockerfile: prod.dockerfile @@ -146,7 +141,7 @@ services: - "traefik.http.routers.ui-main.service=ui-main-svc" s3: - image: "docker.io/minio/minio:RELEASE.2023-10-07T15-07-38Z" + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" container_name: fmtm_s3 environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY} @@ -159,3 +154,9 @@ services: - fmtm-net command: minio server # --console-address ":9090" restart: "unless-stopped" + healthcheck: + test: curl --fail http://localhost:9000/minio/health/live || exit 1 + start_period: 5s + interval: 5s + timeout: 5s + retries: 3 diff --git a/docker-compose.main.yml b/docker-compose.main.yml index caec3d5f85..7f10f510f3 100644 --- a/docker-compose.main.yml +++ b/docker-compose.main.yml @@ -63,7 +63,7 @@ services: - "traefik.http.routers.http_catchall.middlewares=https_redirect" fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ @@ -78,12 +78,7 @@ services: restart: "unless-stopped" api: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" - build: - context: src/backend - target: prod - args: - APP_VERSION: ${API_VERSION} + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_api volumes: - fmtm_logs:/opt/logs @@ -107,7 +102,7 @@ services: - "traefik.http.routers.api.service=api-svc" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/backend:${GIT_BRANCH}" container_name: fmtm_migrations depends_on: - fmtm-db @@ -119,7 +114,7 @@ services: restart: "on-failure:3" ui: - image: "ghcr.io/hotosm/fmtm/frontend:${FRONTEND_MAIN_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/frontend:${GIT_BRANCH}" build: context: src/frontend dockerfile: prod.dockerfile diff --git a/docker-compose.noodk.yml b/docker-compose.noodk.yml index 4919e7c5a0..49e9246335 100644 --- a/docker-compose.noodk.yml +++ b/docker-compose.noodk.yml @@ -30,7 +30,7 @@ networks: services: fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ @@ -70,14 +70,14 @@ services: restart: "unless-stopped" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_VERSION}-${GIT_BRANCH}" + image: "ghcr.io/hotosm/fmtm/backend:debug" container_name: fmtm_migrations depends_on: - fmtm-db env_file: - .env networks: - - fmtm-net + - fmtm-dev entrypoint: ["/migrate-entrypoint.sh"] restart: "on-failure:3" @@ -106,7 +106,7 @@ services: restart: "unless-stopped" s3: - image: "docker.io/minio/minio:RELEASE.2023-10-07T15-07-38Z" + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" container_name: fmtm_s3 environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm} @@ -121,3 +121,9 @@ services: - fmtm-dev command: minio server # --console-address ":9090" restart: "unless-stopped" + healthcheck: + test: curl --fail http://localhost:9000/minio/health/live || exit 1 + start_period: 5s + interval: 5s + timeout: 5s + retries: 3 diff --git a/docker-compose.yml b/docker-compose.yml index 8023185011..75e3b57b8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ networks: services: fmtm-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: fmtm_db volumes: - fmtm_db_data:/var/lib/postgresql/data/ @@ -46,7 +46,7 @@ services: restart: "unless-stopped" api: - image: "ghcr.io/hotosm/fmtm/backend:${API_TAG_OVERRIDE:-debug}" + image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" build: context: src/backend target: debug-with-odk @@ -80,7 +80,7 @@ services: restart: "unless-stopped" migrations: - image: "ghcr.io/hotosm/fmtm/backend:${API_TAG_OVERRIDE:-debug}" + image: "ghcr.io/hotosm/fmtm/backend:${TAG_OVERRIDE:-debug}" container_name: fmtm_migrations depends_on: - fmtm-db @@ -116,7 +116,7 @@ services: restart: "unless-stopped" central-db: - image: "postgis/postgis:14-3.3-alpine" + image: "postgis/postgis:${POSTGIS_TAG:-14-3.3-alpine}" container_name: central_db volumes: - central_db_data:/var/lib/postgresql/data/ @@ -131,11 +131,11 @@ services: restart: "unless-stopped" central: - image: "ghcr.io/hotosm/fmtm/odkcentral:v2023.2.1" + image: "ghcr.io/hotosm/fmtm/odkcentral:${ODK_CENTRAL_TAG:-v2023.4.0}" build: context: odkcentral/api args: - ODK_CENTRAL_VERSION: v2023.2.1 + ODK_CENTRAL_TAG: v2023.4.0 container_name: central_api depends_on: - central-db @@ -156,6 +156,7 @@ services: - EMAIL_IGNORE_TLS=${EMAIL_IGNORE_TLS:-true} - EMAIL_USER=${EMAIL_USER:-''} - EMAIL_PASSWORD=${EMAIL_PASSWORD:-''} + - OIDC_ENABLED=${OIDC_ENABLED:-false} - SENTRY_ORG_SUBDOMAIN=${SENTRY_ORG_SUBDOMAIN:-o130137} - SENTRY_KEY=${SENTRY_KEY:-3cf75f54983e473da6bd07daddf0d2ee} - SENTRY_PROJECT=${SENTRY_PROJECT:-1298632} @@ -163,17 +164,10 @@ services: - "8383:8383" networks: - fmtm-dev - command: - [ - "./wait-for-it.sh", - "${CENTRAL_DB_HOST:-central-db}:5432", - "--", - "./init-user-and-start.sh", - ] restart: "unless-stopped" central-proxy: - image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:latest" + image: "ghcr.io/hotosm/fmtm/odkcentral-proxy:${ODK_CENTRAL_TAG:-v2023.4.0}" build: context: odkcentral/proxy container_name: central_proxy @@ -184,7 +178,7 @@ services: restart: "unless-stopped" s3: - image: "docker.io/minio/minio:RELEASE.2023-10-07T15-07-38Z" + image: "docker.io/minio/minio:${MINIO_TAG:-RELEASE.2023-10-07T15-07-38Z}" container_name: fmtm_s3 environment: MINIO_ROOT_USER: ${S3_ACCESS_KEY:-fmtm} @@ -199,3 +193,9 @@ services: - fmtm-dev command: minio server # --console-address ":9090" restart: "unless-stopped" + healthcheck: + test: curl --fail http://localhost:9000/minio/health/live || exit 1 + start_period: 5s + interval: 5s + timeout: 5s + retries: 3 diff --git a/gen-env.sh b/gen-env.sh index 067b27b9dd..f876c85131 100644 --- a/gen-env.sh +++ b/gen-env.sh @@ -190,8 +190,6 @@ echo echo "Generating dotenv file ${DOTENV_NAME}" echo "### ODK Central ###" -# FIXME set central version via command line -echo "ODK_CENTRAL_VERSION=v2023.2.1" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_URL=${ODK_CENTRAL_URL}" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_USER=${ODK_CENTRAL_USER}" >> "${DOTENV_NAME}" echo "ODK_CENTRAL_PASSWD=${ODK_CENTRAL_PASSWD}" >> "${DOTENV_NAME}" diff --git a/odkcentral/api/Dockerfile b/odkcentral/api/Dockerfile index 1aaff7e750..c8898da9de 100644 --- a/odkcentral/api/Dockerfile +++ b/odkcentral/api/Dockerfile @@ -15,7 +15,7 @@ # along with FMTM. If not, see . # -ARG node_version=16.19.1 +ARG node_version=18 @@ -24,8 +24,8 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ "jq" \ && rm -rf /var/lib/apt/lists/* -ARG ODK_CENTRAL_VERSION=${ODK_CENTRAL_VERSION} -RUN git clone --depth 1 --branch ${ODK_CENTRAL_VERSION} \ +ARG ODK_CENTRAL_TAG +RUN git clone --depth 1 --branch ${ODK_CENTRAL_TAG} \ "https://github.com/getodk/central.git" \ && cd central && git submodule update --init @@ -35,36 +35,38 @@ FROM docker.io/node:${node_version} WORKDIR /usr/odk -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODENAME=\K\w+' /etc/os-release)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ - curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg && \ - apt-get update && \ - apt-get install -y cron gettext postgresql-client-14 - COPY --from=repo central/files/service/crontab /etc/cron.d/odk - -COPY --from=repo central/server/package*.json ./ - -RUN npm clean-install --omit=dev --legacy-peer-deps --no-audit --fund=false --update-notifier=false -RUN npm install pm2@5.2.2 -g - -COPY --from=repo central/server/ ./ COPY --from=repo central/files/service/scripts/ ./ -COPY --from=repo central/files/service/pm2.config.js ./ - COPY --from=repo central/files/service/config.json.template /usr/share/odk/ COPY --from=repo central/files/service/odk-cmd /usr/bin/ +# Add entrypoint script to init user +COPY init-user-and-start.sh / +# package.json must be added and installed prior to final COPY +COPY --from=repo central/server/package*.json ./ -# Required to start via entrypoint -RUN mkdir /etc/secrets sentry-versions \ +# Install system deps +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ $(grep -oP 'VERSION_CODENAME=\K\w+' /etc/os-release)-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list && \ + curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg && \ + apt-get update && \ + apt-get install -y cron wait-for-it gettext postgresql-client-14 netcat-traditional \ + # Install node_modules + && npm clean-install --omit=dev --legacy-peer-deps --no-audit \ + --fund=false --update-notifier=false \ + # Required to start via entrypoint + && mkdir /etc/secrets sentry-versions \ && echo 'jhs9udhy987gyds98gfyds98f' > /etc/secrets/enketo-api-key \ && echo '1' > sentry-versions/server \ && echo '1' > sentry-versions/central \ - && echo '1' > sentry-versions/client + && echo '1' > sentry-versions/client \ + # Set entrypoint executable + && chmod +x /init-user-and-start.sh -# Add entrypoint script to init user -COPY ./init-user-and-start.sh ./ -RUN chmod +x ./init-user-and-start.sh -CMD ["./wait-for-it.sh", "central-db:5432", \ - "--", "./init-user-and-start.sh"] +# Add remaining files after deps installed +COPY --from=repo central/server/ ./ +ENTRYPOINT ["/init-user-and-start.sh"] EXPOSE 8383 + +# Add Healthcheck +HEALTHCHECK --start-period=10s --interval=5s --retries=10 \ + CMD nc -z localhost 8383 || exit 1 diff --git a/odkcentral/api/init-user-and-start.sh b/odkcentral/api/init-user-and-start.sh index 513ec63729..bb421ed25d 100644 --- a/odkcentral/api/init-user-and-start.sh +++ b/odkcentral/api/init-user-and-start.sh @@ -1,5 +1,10 @@ +#!/bin/bash + set -eo pipefail +# Wait for database to be available +wait-for-it "${CENTRAL_DB_HOST:-central-db}:5432" + ### Init, generate config, migrate db ### echo "Stripping pm2 exec command from start-odk.sh script (last 2 lines)" head -n -2 ./start-odk.sh > ./init-odk-db.sh @@ -9,24 +14,14 @@ echo "Running ODKCentral start script to init environment and migrate DB" echo "The server will not start on this run" ./init-odk-db.sh - ### Create admin user ### -echo "Creating test user ${SYSADMIN_EMAIL} with password ${SYSADMIN_PASSWD}" +echo "Creating test user ${SYSADMIN_EMAIL} with password ***${SYSADMIN_PASSWD: -3}" echo "${SYSADMIN_PASSWD}" | odk-cmd --email "${SYSADMIN_EMAIL}" user-create || true echo "Elevating user to admin" odk-cmd --email "${SYSADMIN_EMAIL}" user-promote || true - -### Run server ### -MEMTOT=$(vmstat -s | grep 'total memory' | awk '{ print $1 }') -if [ "$MEMTOT" -gt "1100000" ] -then - export WORKER_COUNT=4 -else - export WORKER_COUNT=1 -fi -echo "using $WORKER_COUNT worker(s) based on available memory ($MEMTOT).." - -echo "Starting server" -exec pm2-runtime ./pm2.config.js +### Run server (hardcode WORKER_COUNT=1 for dev) ### +export WORKER_COUNT=1 +echo "Starting server." +exec npx pm2-runtime ./pm2.config.js diff --git a/odkcentral/proxy/Dockerfile b/odkcentral/proxy/Dockerfile index 5c1fd8fb54..efe2803637 100644 --- a/odkcentral/proxy/Dockerfile +++ b/odkcentral/proxy/Dockerfile @@ -23,3 +23,6 @@ RUN rm -rf ./* /etc/nginx/conf.d/default.conf /etc/nginx/nginx.conf COPY . /etc/nginx RUN cat /etc/nginx/central.crt /etc/nginx/ca.crt \ >> /etc/nginx/central-fullchain.crt +# Add Healthcheck +HEALTHCHECK --start-period=5s --interval=5s --retries=8 \ + CMD nc -z localhost 443 || exit 1 diff --git a/src/backend/Dockerfile b/src/backend/Dockerfile index 69d4f73844..c96a28f7c0 100644 --- a/src/backend/Dockerfile +++ b/src/backend/Dockerfile @@ -18,9 +18,11 @@ ARG PYTHON_IMG_TAG=3.10 FROM docker.io/python:${PYTHON_IMG_TAG}-slim-bookworm as base ARG APP_VERSION +ARG COMMIT_REF ARG PYTHON_IMG_TAG ARG MAINTAINER=admin@hotosm.org LABEL org.hotosm.fmtm.app-version="${APP_VERSION}" \ + org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ org.hotosm.fmtm.python-img-tag="${PYTHON_IMG_TAG}" \ org.hotosm.fmtm.maintainer="${MAINTAINER}" \ org.hotosm.fmtm.api-port="8000" @@ -120,6 +122,9 @@ VOLUME /opt/tiles VOLUME /opt/app/images # Change to non-root user USER appuser +# Add Healthcheck +HEALTHCHECK --start-period=10s --interval=5s --retries=8 --timeout=5s \ + CMD curl --fail http://localhost:8000 || exit 1 diff --git a/src/backend/app-entrypoint.sh b/src/backend/app-entrypoint.sh index f6ecf5af64..8f4c4e20b6 100644 --- a/src/backend/app-entrypoint.sh +++ b/src/backend/app-entrypoint.sh @@ -24,7 +24,7 @@ wait_for_s3() { retry_interval=5 for ((i = 0; i < max_retries; i++)); do - if curl --silent -I ${S3_ENDPOINT} >/dev/null; then + if curl --silent -I ${S3_ENDPOINT:-http://s3:9000} >/dev/null; then echo "S3 is available." return 0 # S3 is available, exit successfully fi diff --git a/src/backend/app/config.py b/src/backend/app/config.py index 7ceb234468..47334aa225 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -20,7 +20,7 @@ from functools import lru_cache from typing import Any, Optional, Union -from pydantic import FieldValidationInfo, PostgresDsn, field_validator +from pydantic import PostgresDsn, ValidationInfo, field_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -41,7 +41,7 @@ class Settings(BaseSettings): def assemble_cors_origins( cls, val: Union[str, list[str]], - info: FieldValidationInfo, + info: ValidationInfo, ) -> Union[list[str], str]: """Build and validate CORS origins list. @@ -82,7 +82,7 @@ def assemble_cors_origins( @field_validator("FMTM_DB_URL", mode="after") @classmethod - def assemble_db_connection(cls, v: Optional[str], info: FieldValidationInfo) -> Any: + def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any: """Build Postgres connection from environment variables.""" if isinstance(v, str): return v diff --git a/src/frontend/prod.dockerfile b/src/frontend/prod.dockerfile index 1d30372ce4..6794ff4151 100644 --- a/src/frontend/prod.dockerfile +++ b/src/frontend/prod.dockerfile @@ -2,11 +2,13 @@ FROM docker.io/node:18 as builder ARG MAINTAINER=admin@hotosm.org ARG APP_VERSION +ARG COMMIT_REF ARG VITE_API_URL ENV VITE_API_URL="${VITE_API_URL}" LABEL org.hotosm.fmtm.app-name="fmtm-frontend" \ org.hotosm.fmtm.app-version="${APP_VERSION}" \ + org.hotosm.fmtm.git-commit-ref="${COMMIT_REF:-none}" \ org.hotosm.fmtm.maintainer="${MAINTAINER}" \ org.hotosm.fmtm.api-url="${VITE_API_URL}"