Merge pull request #225 from hotosm/enhance/dockerfile

fix(dockerfile): Enhance building speed on source code changes

Dockerfile

@@ -5,7 +5,31 @@ FROM docker.io/python:${PYTHON_VERSION}-slim-bookworm as base
 ARG [email protected]
 ENV DEBIAN_FRONTEND=noninteractive
 
-FROM base as builder
+FROM base as runner
+
+WORKDIR /home/appuser
+ENV PIP_NO_CACHE_DIR=1
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/home/appuser/.local/bin:$PATH"
+ENV PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_VERSION/site-packages"
+
+# Install runtime dependencies
+RUN apt-get update \
+    && apt-get -y upgrade \
+    && apt-get --no-install-recommends -y install libpq5 gdal-bin \
+    && apt-get -y autoremove \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+
+FROM ghcr.io/hotosm/tippecanoe:main as tippecanoe-builder
+
+FROM runner as with-tippecanoe
+COPY --from=tippecanoe-builder /usr/local/bin/tippecanoe* /usr/local/bin/
+COPY --from=tippecanoe-builder /usr/local/bin/tile-join /usr/local/bin/
+
+# Builder stage , python dependencies and project setup
+FROM base as python-builder
 
 ENV PIP_NO_CACHE_DIR=1
 ENV PYTHONUNBUFFERED=1
@@ -23,49 +47,30 @@ COPY pyproject.toml .
 COPY requirements.txt .
 COPY README.md .
 COPY LICENSE .
+
 RUN pip install --user --no-cache-dir --upgrade pip setuptools wheel\
     && pip install --user --no-cache-dir GDAL=="$(gdal-config --version)" \
     && pip install --user --no-cache-dir -r requirements.txt
 
-RUN pip install --user -e .
+RUN python setup.py install --user
 
-FROM base as runner
-WORKDIR /home/appuser
-ENV PIP_NO_CACHE_DIR=1
-ENV PYTHONUNBUFFERED=1
-ENV PATH="/home/appuser/.local/bin:$PATH"
-ENV PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_VERSION/site-packages"
 
-RUN apt-get update \
-    && apt-get -y upgrade \
-    && apt-get --no-install-recommends -y install libpq5 gdal-bin \
-    && apt-get -y autoremove \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
+FROM with-tippecanoe as prod
+COPY --from=python-builder /root/.local /home/appuser/.local
 
-COPY --from=builder /root/.local /home/appuser/.local
-COPY README.md .
+RUN useradd --system --uid 900 --home-dir /home/appuser --shell /bin/false appuser \
+    && chown -R appuser:appuser /home/appuser
 
-# Enable this if you are using config.txt
-# COPY config.txt ./config.txt
+USER appuser
 
+# API and source code, changes here don't invalidate previous layers , You can overwrite this block with -v
+
+# Copy config.txt if you have your configuration setup in config
+# COPY config.txt .
+COPY README.md .
 COPY setup.py .
 COPY pyproject.toml .
 COPY API/ ./API/
 COPY src/ ./src/
-# Use a separate stage to pull the tippecanoe image
-FROM ghcr.io/hotosm/tippecanoe:main as tippecanoe-builder
-
-FROM runner as prod
-
-# Copy tippecanoe binaries from the tippecanoe stage
-COPY --from=tippecanoe-builder /usr/local/bin/tippecanoe* /usr/local/bin/
-COPY --from=tippecanoe-builder /usr/local/bin/tile-join /usr/local/bin/
-
-RUN useradd --system --uid 900 --home-dir /home/appuser --shell /bin/false appuser \
-    && chown -R appuser:appuser /home/appuser
-
-USER appuser
 
-# CMD ["/bin/bash"]
-CMD ["uvicorn", "API.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000", "--no-use-colors", "--proxy-headers"]
+CMD ["uvicorn", "API.main:app", "--reload", "--host", "0.0.0.0", "--port", "8000", "--no-use-colors", "--proxy-headers"]

docker-compose-config.txt

@@ -0,0 +1,21 @@
+[DB]
+PGHOST=pgsql
+PGUSER=postgres
+PGPASSWORD=admin
+PGDATABASE=raw
+PGPORT=5432
+
+[API_CONFIG]
+RATE_LIMITER_STORAGE_URI=redis://redis:6379
+
+[CELERY]
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+
+[OAUTH]
+OSM_CLIENT_ID=
+OSM_CLIENT_SECRET=
+OSM_URL=https://www.openstreetmap.org
+OSM_PERMISSION_SCOPE=read_prefs
+LOGIN_REDIRECT_URI=http://127.0.0.1:8000/v1/auth/callback
+APP_SECRET_KEY=replace_this_with_your_trusted_secret_key

docker-compose.yml

@@ -1,58 +1,78 @@
-version: '3.8'
+version: "3.8"
 
 services:
-
   postgres:
-      restart: always
-      image: postgis/postgis
-      container_name: pgsql
-      environment:
-        - POSTGRES_DB=raw
-        - POSTGRES_USER=postgres
-        - POSTGRES_PASSWORD=admin
-      ports:
-        - '5434:5432'
-      volumes:
-        - ./postgres-data:/var/lib/postgresql/data
-        - ./tests/fixtures/pokhara.sql:/docker-entrypoint-initdb.d/pokhara.sql
+    restart: always
+    image: postgis/postgis
+    container_name: pgsql
+    environment:
+      - POSTGRES_DB=raw
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=admin
+    ports:
+      - "5434:5432"
+    volumes:
+      - ./postgres-data:/var/lib/postgresql/data
+      - ./tests/fixtures/pokhara.sql:/docker-entrypoint-initdb.d/pokhara.sql
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres -d raw"]
+      interval: 60s
+      timeout: 30s
+      retries: 5
+      start_period: 10s
 
   redis:
     image: redis
     container_name: redis
     ports:
-     - "6379:6379"
+      - "6379:6379"
 
-  app:
+  api:
     build: .
-    container_name: api
+    container_name: rawdataapi
     command: uvicorn API.main:app --reload --host 0.0.0.0 --port 8000 --no-use-colors --proxy-headers
     ports:
       - 8000:8000
     volumes:
-      - .:/app
+      - "./API:/home/appuser/API"
+      - "./src:/home/appuser/src"
+      - "./docker-compose-config.txt:/home/appuser/config.txt"
     depends_on:
-      - redis
-      - postgres
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
   worker:
     build: .
-    container_name: worker
-    command: celery --app API.api_worker worker --loglevel=INFO
+    container_name: rawdata-worker
+    command: celery --app API.api_worker worker --loglevel=INFO --queues="raw_ondemand" -n 'default_worker'
     volumes:
-      - .:/app
+      - "./API:/home/appuser/API"
+      - "./src:/home/appuser/src"
+      - "./docker-compose-config.txt:/home/appuser/config.txt"
     depends_on:
-      - app
-      - redis
-      - postgres
+      api:
+        condition: service_started
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy
 
-
-  worker-dashboard:
+  flower:
     build: .
-    container_name: flower
-    command:  celery --broker=redis://redis:6379// --app API.api_worker flower --address=0.0.0.0 --port=5000
+    container_name: rawdata-flower
+    command: celery --broker=redis://redis:6379// --app API.api_worker flower --port=5555 --queues="raw_daemon,raw_ondemand"
     ports:
-      - 5000:5000
+      - 5555:5555
+    volumes:
+      - "./API:/home/appuser/API"
+      - "./src:/home/appuser/src"
+      - "./docker-compose-config.txt:/home/appuser/config.txt"
     depends_on:
-      - app
-      - redis
-      - worker
+      api:
+        condition: service_started
+      redis:
+        condition: service_started
+      postgres:
+        condition: service_healthy

docs/src/installation/docker.md

@@ -12,7 +12,7 @@ cd raw-data-api
 ## Configurations
 
 ### config.txt approach
-- Create `config.txt` inside / folder. You can use any of the appropriate commands below or you use your familiar methods in your code editor/file explorer.
+- Create `config.txt` inside / folder. You can use any of the appropriate commands below or you use your familiar methods in your code editor/file explorer. (if you are using `docker-compose` , you can edit `docker-compose-config.txt`)
 
 ```
 touch config.txt #Linux
@@ -24,27 +24,10 @@ echo >> config.txt #Windows without WSL
 if you prefer configurations as env variables you can put them in `.env` and pass it to dockerfile or export them 
 
 - Database configuration:
-  - To use the default database(with sample data) shipped with the `Dockerfile`, you can update the `config.txt` with the configurations below. [**Recommended**]
-
-  - Make sure you uncomment ```COPY config.txt ./config.txt``` line in Dockerfile while using `config.txt`
+  - To use the default database(with sample data) , Run docker compsoe and  update the `docker-compose-config.txt` 
 
   - To use a local postgres (with postgis enabled) database, you can follow the instruction on how to set it up with raw data [here](./configurations.md). or export them as system env variables
 
-```
-[DB]
-PGHOST=pgsql
-PGUSER=postgres
-PGPASSWORD=admin
-PGDATABASE=raw
-PGPORT=5432
-
-[API_CONFIG]
-RATE_LIMITER_STORAGE_URI=redis://redis:6379
-
-[CELERY]
-CELERY_BROKER_URL=redis://redis:6379/0
-CELERY_RESULT_BACKEND=redis://redis:6379/0
-```
 
 - OSM Authentication:
 
@@ -64,31 +47,55 @@ docker-compose up -d --build
 
 OR 
 
-### Run Docker without docker compose 
+### Run Docker without docker compose for development
 
 - Build your image 
 ```
 docker build -t rawdataapi:latest . 
 ```
 - Run API 
 ```
-docker run -d -p 8000:8000 --name rawdatapi rawdataapi:latest
+docker run --name rawdataapi -p 8000:8000 \
+    -v "$(pwd)/API:/home/appuser/API" \
+    -v "$(pwd)/src:/home/appuser/src" \
+    -v "$(pwd)/config.txt:/home/appuser/config.txt" \
+    rawdataapi:latest \
+    uvicorn API.main:app --reload --host "0.0.0.0" --port "8000" --no-use-colors --proxy-headers
 ```
-Run container with `.env` file 
 
+- Run Workers 
 ```
-docker run --env-file ./.env -d -p 8000:8000 --name rawdatapi rawdataapi:latest
+docker run --name rawdata-worker \
+    -v "$(pwd)/API:/home/appuser/API" \
+    -v "$(pwd)/src:/home/appuser/src" \
+    -v "$(pwd)/config.txt:/home/appuser/config.txt" \
+    rawdataapi:latest \
+    celery --app API.api_worker worker --loglevel=INFO --queues="raw_ondemand" -n 'default_worker'
 ```
-
-- Run Workers 
+- Run Flower on port 5555
 ```
-docker run -it rawdataapi:latest celery --app API.api_worker worker --loglevel=INFO --queues="raw_ondemand" -n 'default_worker'
+docker run --name rawdata-flower -p 5555:5555 \
+    -v "$(pwd)/API:/home/appuser/API" \
+    -v "$(pwd)/src:/home/appuser/src" \
+    -v "$(pwd)/config.txt:/home/appuser/config.txt" \
+    rawdataapi:latest \
+    celery --broker=redis://redis:6379// --app API.api_worker flower --port=5555 --queues="raw_daemon,raw_ondemand"
 ```
-Followi similar for flower
+
+**Development instruction:** 
+If you are running Dockerfile only for the API , Have postgresql redis installed on your machine directly then you should change following : 
+
+- Change --broker Host address in flower command (You can use redis if it is docker compsoe container or use `redis://host.docker.internal:6379/0` if you want API container to connect to your localhsot , Follow #troubleshoot section for more)
+- Change DB Host & Celery broker url accordingly with the same logic 
+
+
+**Note:**
+
+In above example we have attached our working dir to containers along with config.txt for efficiency in development environment only . It is recommended to use proper docker copy as stated in dockerfile and system environement variables instead of config.txt in Production
 
 ## Check the servers
 
-By default, Uvicorn would be running on port [8000](http://127.0.0.1:8000/v1/docs), Redis on default port(i.e 6379), Celery with a worker and Flower on port [5000](http://127.0.0.1:5000/).
+By default, Uvicorn would be running on port [8000](http://127.0.0.1:8000/v1/docs), Redis on default port(i.e 6379), Celery with a worker and Flower on port [5555](http://127.0.0.1:5555/).
 
 - API Documentation
 
@@ -107,10 +114,10 @@ API docs will be displayed like this upon successfull server startup
 Vist the route below to access the Flower dashboard
 
 ```
-http://127.0.0.1:5000/
+http://127.0.0.1:5555/
 ```
 
-Flower [dashboard](http://127.0.0.1:5000/) will look like this on successfull installation with a worker online.
+Flower [dashboard](http://127.0.0.1:5555/) will look like this on successfull installation with a worker online. (Change the port accordingly to your command)
 
 ![image](https://user-images.githubusercontent.com/36752999/191813613-3859522b-ea68-4370-87b2-ebd1d8880d80.png)
 
@@ -120,7 +127,7 @@ Flower [dashboard](http://127.0.0.1:5000/) will look like this on successfull in
 
 **NOTE:** If any of the solutions provided below does not work for you, please don't hesistate to open an issue.
 
-- If you can't connect to your local postgres database from `export-tool-api`.
+- If you can't connect to your local postgres database from `raw-data-api`.
 
   Since _export-tool-api_ is running in docker containers, it means if you setup your `config.txt` file to use a local postgres database on your machine, the database port may not be accessible as `localhost` from the docker containers. To solve this, docker needs to connect to your local network. Try any of these hacks to troubleshoot the issue: